From 82343e66b80dfcb5b744f5992d02bc8ffa656941 Mon Sep 17 00:00:00 2001
From: Moises Sacal <moisbo@gmail.com>
Date: Wed, 29 May 2019 15:51:52 +1000
Subject: [PATCH] added a commit-to-solr runner to test catalog solr and commit
to a running solr for
https://code.research.uts.edu.au/eresearch/rdm/issues/912
---
.idea/workspace.xml | 175 +++++++++++++++++++++++++++---------------
commit-to-solr.js | 143 ++++++++++++++++++++++++++++++++++
config.json | 8 ++
fields.json | 65 ++++++++++++++++
lib/CatalogSolr.js | 35 ++++-----
package-lock.json | 109 +++++++-------------------
package.json | 3 +-
test-data/fields.json | 1 +
test/catalog.spec.js | 14 +---
9 files changed, 378 insertions(+), 175 deletions(-)
create mode 100644 commit-to-solr.js
create mode 100644 config.json
create mode 100644 fields.json
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index a67af72..8e69ce4 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,6 +2,14 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
+ <change afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
+ <change afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
+ <change afterPath="$PROJECT_DIR$/fields.json" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/lib/CatalogSolr.js" beforeDir="false" afterPath="$PROJECT_DIR$/lib/CatalogSolr.js" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/test-data/fields.json" beforeDir="false" afterPath="$PROJECT_DIR$/test-data/fields.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/test/catalog.spec.js" beforeDir="false" afterPath="$PROJECT_DIR$/test/catalog.spec.js" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/.tmp/" />
@@ -14,23 +22,41 @@
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
- <splitter split-orientation="horizontal" split-proportion="0.5009346">
+ <splitter split-orientation="horizontal" split-proportion="0.5182222">
<split-first>
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
- <file pinned="false" current-in-tab="true">
+ <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="269">
- <caret line="137" column="63" selection-start-line="137" selection-start-column="63" selection-end-line="137" selection-end-column="63" />
+ <state relative-caret-position="395">
+ <caret line="121" column="6" selection-start-line="121" selection-start-column="6" selection-end-line="121" selection-end-column="6" />
+ </state>
+ </provider>
+ </entry>
+ </file>
+ <file pinned="false" current-in-tab="false">
+ <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="587">
+ <caret line="137" column="31" lean-forward="true" selection-start-line="137" selection-start-column="31" selection-end-line="137" selection-end-column="31" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
+ <entry file="file://$PROJECT_DIR$/config.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="75">
+ <caret line="5" column="18" selection-start-line="5" selection-start-column="18" selection-end-line="5" selection-end-column="18" />
+ </state>
+ </provider>
+ </entry>
+ </file>
+ <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="217">
- <caret line="90" column="17" selection-start-line="90" selection-start-column="17" selection-end-line="90" selection-end-column="17" />
+ <state relative-caret-position="245">
+ <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
</state>
</provider>
</entry>
@@ -60,8 +86,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="308">
- <caret line="62" column="22" selection-start-line="62" selection-start-column="16" selection-end-line="62" selection-end-column="22" />
+ <state relative-caret-position="329">
+ <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
</state>
</provider>
</entry>
@@ -69,8 +95,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/test-data/fields.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="678">
- <caret line="46" column="4" selection-start-line="46" selection-start-column="4" selection-end-line="46" selection-end-column="4" />
+ <state relative-caret-position="409">
+ <caret line="32" column="39" selection-start-line="32" selection-start-column="25" selection-end-line="32" selection-end-column="39" />
</state>
</provider>
</entry>
@@ -78,8 +104,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test-data/CatalogSolr.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="-459">
- <caret line="2" column="29" selection-start-line="2" selection-start-column="22" selection-end-line="2" selection-end-column="29" />
+ <state relative-caret-position="-17">
+ <caret line="27" column="31" selection-start-line="27" selection-start-column="5" selection-end-line="27" selection-end-column="31" />
</state>
</provider>
</entry>
@@ -87,8 +113,8 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="300">
- <caret line="20" column="14" selection-start-line="20" selection-start-column="7" selection-end-line="20" selection-end-column="14" />
+ <state relative-caret-position="915">
+ <caret line="61" column="20" selection-start-line="61" selection-start-column="20" selection-end-line="61" selection-end-column="20" />
</state>
</provider>
</entry>
@@ -101,6 +127,7 @@
<findStrings>
<find>datePublished</find>
<find>geo</find>
+ <find>requestData</find>
</findStrings>
</component>
<component name="Git.Settings">
@@ -116,17 +143,20 @@
<option value="$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_3.js" />
<option value="$PROJECT_DIR$/test-data/CatalogSolr.json" />
- <option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
- <option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/.gitignore" />
+ <option value="$PROJECT_DIR$/fields.json" />
+ <option value="$PROJECT_DIR$/config.json" />
+ <option value="$PROJECT_DIR$/commit-to-solr.js" />
<option value="$PROJECT_DIR$/test/catalog.spec.js" />
+ <option value="$PROJECT_DIR$/test-data/fields.json" />
+ <option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
- <option name="x" value="1442" />
+ <option name="x" value="1440" />
<option name="y" value="-277" />
- <option name="width" value="1918" />
+ <option name="width" value="1920" />
<option name="height" value="1177" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
@@ -147,11 +177,6 @@
<item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
<item name="lib" type="462c0819:PsiDirectoryNode" />
</path>
- <path>
- <item name="solr-catalog" type="b2602c69:ProjectViewProjectNode" />
- <item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
- <item name="test" type="462c0819:PsiDirectoryNode" />
- </path>
<path>
<item name="solr-catalog" type="b2602c69:ProjectViewProjectNode" />
<item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
@@ -172,7 +197,7 @@
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="com.intellij.ide.scratch.LRUPopupBuilder$1/New Scratch File" value="JavaScript" />
- <property name="last_opened_file_path" value="$PROJECT_DIR$/test-data" />
+ <property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs.mocha.mocha_node_package_dir" value="$PROJECT_DIR$/node_modules/mocha" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
@@ -181,6 +206,7 @@
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
+ <recent name="$PROJECT_DIR$" />
<recent name="$PROJECT_DIR$/test-data" />
</key>
</component>
@@ -196,7 +222,10 @@
</list>
</option>
</component>
- <component name="RunManager">
+ <component name="RunManager" selected="Node.js.commit-to-solr">
+ <configuration name="commit-to-solr" type="NodeJSConfigurationType" path-to-js-file="commit-to-solr.js" working-dir="$PROJECT_DIR$">
+ <method v="2" />
+ </configuration>
<configuration name="catalog.spec.js" type="mocha-javascript-test-runner">
<node-interpreter>project</node-interpreter>
<node-options />
@@ -209,6 +238,10 @@
<test-file>$PROJECT_DIR$/test/catalog.spec.js</test-file>
<method v="2" />
</configuration>
+ <list>
+ <item itemvalue="Mocha.catalog.spec.js" />
+ <item itemvalue="Node.js.commit-to-solr" />
+ </list>
</component>
<component name="SvnConfiguration">
<configuration />
@@ -221,62 +254,61 @@
<option name="presentableId" value="Default" />
<updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" />
- <workItem from="1558654598476" duration="36874000" />
+ <workItem from="1558654598476" duration="48948000" />
</task>
<servers />
</component>
<component name="TestHistory">
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 24m 46s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 14h 50m 34s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 25m 32s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 29m 39s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 26m 13s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 29m 51s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 28m 24s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 31m 49s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 31m 09s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 31m 51s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 31m 10s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 33m 16s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 32m 20s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 34m 30s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 32m 46s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 35m 20s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 33m 20s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 35m 32s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
- <history-entry file="catalog_spec_js - 2019.05.29 at 11h 33m 21s.xml">
+ <history-entry file="catalog_spec_js - 2019.05.29 at 15h 35m 38s.xml">
<configuration name="catalog.spec.js" configurationId="mocha-javascript-test-runner" />
</history-entry>
</component>
<component name="TimeTrackingManager">
- <option name="totallyTimeSpent" value="67352000" />
+ <option name="totallyTimeSpent" value="79426000" />
</component>
<component name="ToolWindowManager">
<frame x="1440" y="-277" width="1920" height="1177" extended-state="6" />
- <editor active="true" />
<layout>
- <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.15481833" />
+ <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.15481833" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" weight="0.3299539" />
- <window_info anchor="bottom" id="Debug" order="3" weight="0.35668203" />
+ <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.27834103" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="8" />
- <window_info anchor="bottom" id="Terminal" order="9" visible="true" weight="0.23870967" />
+ <window_info anchor="bottom" id="Terminal" order="9" weight="0.23821656" />
<window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
<window_info anchor="right" id="Commander" order="0" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
@@ -306,52 +338,73 @@
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json">
+ <entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="300">
- <caret line="20" column="14" selection-start-line="20" selection-start-column="7" selection-end-line="20" selection-end-column="14" />
+ <state relative-caret-position="30">
+ <caret line="2" column="21" lean-forward="true" selection-start-line="2" selection-start-column="21" selection-end-line="2" selection-end-column="21" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
+ <entry file="file://$PROJECT_DIR$/fields.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="164">
+ <caret line="29" column="20" lean-forward="true" selection-start-line="29" selection-start-column="20" selection-end-line="29" selection-end-column="20" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="308">
- <caret line="62" column="22" selection-start-line="62" selection-start-column="16" selection-end-line="62" selection-end-column="22" />
+ <state relative-caret-position="75">
+ <caret line="5" column="18" selection-start-line="5" selection-start-column="18" selection-end-line="5" selection-end-column="18" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test-data/CatalogSolr.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="-459">
- <caret line="2" column="29" selection-start-line="2" selection-start-column="22" selection-end-line="2" selection-end-column="29" />
+ <state relative-caret-position="-17">
+ <caret line="27" column="31" selection-start-line="27" selection-start-column="5" selection-end-line="27" selection-end-column="31" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/fields.json">
+ <entry file="file://$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="678">
- <caret line="46" column="4" selection-start-line="46" selection-start-column="4" selection-end-line="46" selection-end-column="4" />
+ <state relative-caret-position="915">
+ <caret line="61" column="20" selection-start-line="61" selection-start-column="20" selection-end-line="61" selection-end-column="20" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
+ <entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="217">
- <caret line="90" column="17" selection-start-line="90" selection-start-column="17" selection-end-line="90" selection-end-column="17" />
+ <state relative-caret-position="395">
+ <caret line="121" column="6" selection-start-line="121" selection-start-column="6" selection-end-line="121" selection-end-column="6" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/.gitignore">
+ <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="30">
- <caret line="2" column="21" lean-forward="true" selection-start-line="2" selection-start-column="21" selection-end-line="2" selection-end-column="21" />
+ <state relative-caret-position="587">
+ <caret line="137" column="31" lean-forward="true" selection-start-line="137" selection-start-column="31" selection-end-line="137" selection-end-column="31" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
+ <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="329">
+ <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/test-data/fields.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="409">
+ <caret line="32" column="39" selection-start-line="32" selection-start-column="25" selection-end-line="32" selection-end-column="39" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="269">
- <caret line="137" column="63" selection-start-line="137" selection-start-column="63" selection-end-line="137" selection-end-column="63" />
+ <state relative-caret-position="245">
+ <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
</state>
</provider>
</entry>
diff --git a/commit-to-solr.js b/commit-to-solr.js
new file mode 100644
index 0000000..7276d01
--- /dev/null
+++ b/commit-to-solr.js
@@ -0,0 +1,143 @@
+const axios = require('axios');
+const _ = require('lodash');
+const yargs = require('yargs');
+const CatalogSolr = require('./lib/CatalogSolr');
+const fs = require('fs-extra');
+
+const argv = yargs['argv'];
+const configPath = argv.config || './config.json';
+if (!fs.existsSync(configPath)) {
+ console.error(`Please provide a valid config file path: ${configPath}`);
+ process.exit(1);
+}
+const configJson = require(configPath);
+const sourcePath = _.endsWith(configJson.source, '/') ? configJson.source : `${configJson.source}/`;
+const solrUpdate = configJson['solrUpdate'] || '';
+const fieldConfig = require(configJson['fields']);
+const logLevel = configJson['logLevel'] || 4;
+const waitPeriod = configJson['waitPeriod'] || 0;
+
+const catalog = new CatalogSolr();
+catalog.setConfig(fieldConfig);
+
+const sleep = ms => new Promise((r, j) => {
+ console.log('Waiting for ' + ms + ' seconds');
+ setTimeout(r, ms * 1000);
+});
+
+function commitDocs(URI) {
+ return axios({
+ url: solrUpdate + URI,
+ method: 'get',
+ responseType: 'json',
+ headers: {
+ 'Content-Type': 'application/json; charset=utf-8'
+ }
+ });
+}
+
+function updateDocs(coreObjects) {
+ return axios({
+ url: solrUpdate + '/docs',
+ method: 'post',
+ data: coreObjects,
+ responseType: 'json',
+ headers: {
+ 'Content-Type': 'application/json; charset=utf-8'
+ }
+ });
+}
+
+function recordsArray(sourcePath) {
+ const records = [];
+ _.each(fs.readdirSync(sourcePath, {encoding: 'utf-8', withFileTypes: true}), (dirEnt) => {
+ // fs.dirEnt needs node version 10 +
+ if (dirEnt.isDirectory()) {
+ const entryPath = `${sourcePath}${dirEnt.name}/CATALOG.json`;
+ if (fs.existsSync(entryPath)) {
+ const entryJson = require(entryPath);
+ records.push(entryJson);
+ if (logLevel >= 4) console.log(`Added: ${entryPath}`);
+ } else {
+ console.error(`CATALOG.json missing: ${entryPath}`);
+ }
+ } else {
+ if (logLevel >= 4) console.log(`Ignoring, not a directory: ${dirEnt.name}`);
+ }
+ });
+ return records;
+}
+
+function createCatalogSolr(ca) {
+ //Peter's idea is to convert everything into an array then it is safer to work to convert
+ const graph = _.each(ca['@graph'], (g) => {
+ return catalog.ensureObjArray(g);
+ });
+
+ const catalogSolr = {};
+ _.each(fieldConfig, (field, name) => {
+ let graphElement = _.filter(graph, (g) => {
+ return _.find(g['@type'], (gg) => gg === name) ? g : undefined;
+ });
+ if (graphElement) {
+ _.each(graphElement, (ge) => {
+ if (Array.isArray(catalogSolr[name])) {
+ catalogSolr[name].push(catalog.getGraphElement(fieldConfig[name], graph, ge));
+ } else {
+ catalogSolr[name] = [catalog.getGraphElement(fieldConfig[name], graph, ge)];
+ }
+ });
+ }
+ });
+
+ return catalogSolr;
+}
+
+let records = [];
+if (fs.existsSync(sourcePath)) {
+ records = recordsArray(sourcePath);
+} else {
+ console.error(`Source path doesn't exist: ${sourcePath}`);
+ process.exit(1);
+}
+
+catalogs = [];
+
+records.forEach((r) => {
+ const solrObj = createCatalogSolr(r);
+ if (solrObj) {
+ if (solrObj.Dataset) {
+ solrObj.Dataset.forEach((c) => {
+ catalogs.push(c);
+ });
+ }
+ if (solrObj.Person) {
+ solrObj.Person.forEach((c) => {
+ catalogs.push(c);
+ });
+ }
+ }
+
+});
+
+const fakePromises = [1];
+
+fakePromises.reduce((promise, p, index) => {
+ return promise.then(() => {
+ return updateDocs(catalogs).then(async () => {
+ if (waitPeriod) {
+ const waited = await sleep(waitPeriod);
+ }
+ console.log('Update docs');
+ if (index >= fakePromises.length - 1) {
+ console.log('run commit');
+ return commitDocs('?commit=true&overwrite=true').then(() => {
+ return Promise.resolve();
+ });
+ }
+ return Promise.resolve();
+ });
+ }).catch((e) => {
+ console.log(e);
+ })
+}, Promise.resolve());
\ No newline at end of file
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..36c2652
--- /dev/null
+++ b/config.json
@@ -0,0 +1,8 @@
+{
+ "source": "/Users/moises/source/peppermint/publication/public/",
+ "ocfl": "/Users/moises/source/peppermint/publication/ocfl/",
+ "solrUpdate": "http://localhost:8983/solr/dataset/update/json",
+ "logLevel": 3,
+ "waitPeriod": 1,
+ "fields": "./fields.json"
+}
diff --git a/fields.json b/fields.json
new file mode 100644
index 0000000..34860a6
--- /dev/null
+++ b/fields.json
@@ -0,0 +1,65 @@
+{
+ "Dataset": {
+ "core": "dataset",
+ "format": "jsonld",
+ "recordTypeName": "dataset",
+ "skip": ["hasPart"],
+ "flatten": {
+ "creator": {
+ "obj": "array"
+ },
+ "GeoCoordinates": {}
+ },
+ "facets": {
+ "datePublished": {
+ "field_suffix": "_Dataset_facet",
+ "trim": true
+ },
+ "keywords": {
+ "trim": true,
+ "field_suffix": "_Dataset_facetmulti",
+ "tokenize": {
+ "delim": ","
+ }
+ },
+ "creator": {
+ "trim": true,
+ "fieldName": "name",
+ "field_suffix": "_Dataset_facetmulti"
+ },
+ "publisher": {
+ "trim": true,
+ "fieldName": "@id",
+ "field_suffix": "_Dataset_facet"
+ },
+ "type": {
+ "trim": true,
+ "field_suffix": "facetmulti",
+ "skip_entry_type_suffix": true,
+ "escape_value": "solr_field"
+ }
+ },
+ "oai-pmh": {
+ "core": "dataset",
+ "set": "datasource:ReDBox"
+ }
+ },
+ "Person": {
+ "core": "dataset",
+ "format": "jsonld",
+ "recordTypeName": "person",
+ "flatten": {
+ "affiliation": {
+ "obj": "array"
+ }
+ },
+ "facets": {
+ "type": {
+ "trim": true,
+ "field_suffix": "_Dataset_facetmulti",
+ "skip_entry_type_suffix": true,
+ "escape_value": "solr_field"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/lib/CatalogSolr.js b/lib/CatalogSolr.js
index f09c3e7..4ee3eec 100644
--- a/lib/CatalogSolr.js
+++ b/lib/CatalogSolr.js
@@ -3,8 +3,7 @@ const assert = require('assert');
class CatalogSolr {
- constructor(jsonld) {
- this.jsonld = jsonld;
+ constructor() {
}
setConfig(config) {
@@ -21,25 +20,11 @@ class CatalogSolr {
return true;
}
- loadFields() {
-
- }
-
- getSimple() {
-
- }
-
enforceSolrFieldNames(k) {
const k1 = k.replace('https?://schema.org/', '');
return k1.replace(/[^a-zA-Z\d_]/, '_');
}
- getFacets() {
- _.each(this.config['facets'], function (facet) {
- console.log(facet);
- });
- }
-
ensureObjArray(graphElement) {
_.forOwn(graphElement, (ge, prop) => {
if (!_.isArray(ge)) {
@@ -52,8 +37,8 @@ class CatalogSolr {
findFacetObject(graph, obj, name, value, facetInfo) {
let key = '';
- // TODO: send _Dataset prefix from function
- if (facetInfo['field_suffix'] === '_Dataset_facetmulti') {
+ // TODO: send _Dataset (core) prefix from function
+ if (facetInfo['field_suffix']) {
if (facetInfo['tokenize']) {
value = value.split(facetInfo['tokenize']['delim']);
}
@@ -105,7 +90,13 @@ class CatalogSolr {
});
if (facetInfo) {
const facetObject = this.findFacetObject(graph, gg, kk, ggg, facetInfo);
- base[facetObject['key']] = facetObject['value'];
+ if (Array.isArray(facetObject)) {
+ facetObject.forEach((fO) => {
+ base[fO['key']] = fO['value'];
+ });
+ } else {
+ base[facetObject['key']] = facetObject['value'];
+ }
}
if (kk === '@type') { //TODO handle this by config!
base.record_type_s = ggg;
@@ -130,12 +121,12 @@ class CatalogSolr {
const objFound = this.getGraphElement(config, graph, found);
if (config.flatten[kk]['obj'] === 'array') {
if (Array.isArray(base[kk])) {
- base[kk].push(JSON.stringify(objFound));
+ base[kk].push(JSON.stringify(objFound).replace(/"/g, '\''));
} else {
- base[kk] = [JSON.stringify(objFound)];
+ base[kk] = [JSON.stringify(objFound).replace(/"/g, '\'')];
}
} else {
- base[kk] = JSON.stringify(objFound)
+ base[kk] = JSON.stringify(objFound).replace(/"/g, '\'')
}
} else {
base[kk] = this.getGraphElement(config, graph, found);
diff --git a/package-lock.json b/package-lock.json
index d91de9a..d11211f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -77,14 +77,12 @@
"ansi-regex": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
- "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
- "dev": true
+ "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg=="
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
- "dev": true,
"requires": {
"color-convert": "^1.9.0"
}
@@ -235,8 +233,7 @@
"camelcase": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
- "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==",
- "dev": true
+ "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg=="
},
"caseless": {
"version": "0.12.0",
@@ -302,7 +299,6 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz",
"integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==",
- "dev": true,
"requires": {
"string-width": "^3.1.0",
"strip-ansi": "^5.2.0",
@@ -337,7 +333,6 @@
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
- "dev": true,
"requires": {
"color-name": "1.1.3"
}
@@ -345,8 +340,7 @@
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
- "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
- "dev": true
+ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
},
"combined-stream": {
"version": "1.0.8",
@@ -389,7 +383,6 @@
"version": "6.0.5",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
"integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
- "dev": true,
"requires": {
"nice-try": "^1.0.4",
"path-key": "^2.0.1",
@@ -502,8 +495,7 @@
"decamelize": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
- "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=",
- "dev": true
+ "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
},
"deep-is": {
"version": "0.1.3",
@@ -595,14 +587,12 @@
"emoji-regex": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
- "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==",
- "dev": true
+ "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA=="
},
"end-of-stream": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
"integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
- "dev": true,
"requires": {
"once": "^1.4.0"
}
@@ -685,7 +675,6 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
"integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==",
- "dev": true,
"requires": {
"cross-spawn": "^6.0.0",
"get-stream": "^4.0.0",
@@ -742,7 +731,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz",
"integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==",
- "dev": true,
"requires": {
"locate-path": "^3.0.0"
}
@@ -820,14 +808,12 @@
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
- "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
- "dev": true
+ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-stream": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
"integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
- "dev": true,
"requires": {
"pump": "^3.0.0"
}
@@ -994,8 +980,7 @@
"invert-kv": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz",
- "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
- "dev": true
+ "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA=="
},
"is-buffer": {
"version": "1.1.6",
@@ -1017,8 +1002,7 @@
"is-fullwidth-code-point": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
- "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
- "dev": true
+ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
},
"is-regex": {
"version": "1.0.4",
@@ -1032,8 +1016,7 @@
"is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
- "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
- "dev": true
+ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
},
"is-symbol": {
"version": "1.0.2",
@@ -1053,8 +1036,7 @@
"isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
- "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
- "dev": true
+ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
},
"isstream": {
"version": "0.1.2",
@@ -1194,7 +1176,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz",
"integrity": "sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA==",
- "dev": true,
"requires": {
"invert-kv": "^2.0.0"
}
@@ -1219,7 +1200,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
"integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==",
- "dev": true,
"requires": {
"p-locate": "^3.0.0",
"path-exists": "^3.0.0"
@@ -1249,7 +1229,6 @@
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
"integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==",
- "dev": true,
"requires": {
"p-defer": "^1.0.0"
}
@@ -1258,7 +1237,6 @@
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz",
"integrity": "sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==",
- "dev": true,
"requires": {
"map-age-cleaner": "^0.1.1",
"mimic-fn": "^2.0.0",
@@ -1283,8 +1261,7 @@
"mimic-fn": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
- "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
- "dev": true
+ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
},
"minimatch": {
"version": "3.0.4",
@@ -1485,8 +1462,7 @@
"nice-try": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
- "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
- "dev": true
+ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ=="
},
"node-environment-flags": {
"version": "1.0.5",
@@ -1514,7 +1490,6 @@
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
"integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
- "dev": true,
"requires": {
"path-key": "^2.0.0"
}
@@ -1578,7 +1553,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
- "dev": true,
"requires": {
"wrappy": "1"
}
@@ -1601,7 +1575,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz",
"integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==",
- "dev": true,
"requires": {
"execa": "^1.0.0",
"lcid": "^2.0.0",
@@ -1617,26 +1590,22 @@
"p-defer": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
- "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=",
- "dev": true
+ "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww="
},
"p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
- "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
- "dev": true
+ "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
},
"p-is-promise": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz",
- "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==",
- "dev": true
+ "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg=="
},
"p-limit": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz",
"integrity": "sha512-pZbTJpoUsCzV48Mc9Nh51VbwO0X9cuPFE8gYwx9BTCt9SF8/b7Zljd2fVgOxhIF/HDTKgpVzs+GPhyKfjLLFRQ==",
- "dev": true,
"requires": {
"p-try": "^2.0.0"
}
@@ -1645,7 +1614,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz",
"integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==",
- "dev": true,
"requires": {
"p-limit": "^2.0.0"
}
@@ -1653,8 +1621,7 @@
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
- "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
- "dev": true
+ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
},
"parse5": {
"version": "3.0.3",
@@ -1668,8 +1635,7 @@
"path-exists": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
- "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=",
- "dev": true
+ "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU="
},
"path-is-absolute": {
"version": "1.0.1",
@@ -1680,8 +1646,7 @@
"path-key": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
- "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=",
- "dev": true
+ "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
},
"path-parse": {
"version": "1.0.6",
@@ -1729,7 +1694,6 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
- "dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
@@ -1865,14 +1829,12 @@
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
- "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
- "dev": true
+ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"require-main-filename": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz",
- "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
- "dev": true
+ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg=="
},
"resolve": {
"version": "1.11.0",
@@ -1904,20 +1866,17 @@
"semver": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
- "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
- "dev": true
+ "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
},
"set-blocking": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
- "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
- "dev": true
+ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
},
"shebang-command": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
"integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
- "dev": true,
"requires": {
"shebang-regex": "^1.0.0"
}
@@ -1925,8 +1884,7 @@
"shebang-regex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
- "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
- "dev": true
+ "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
},
"shelljs": {
"version": "0.8.3",
@@ -1942,8 +1900,7 @@
"signal-exit": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
- "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
- "dev": true
+ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
},
"source-map": {
"version": "0.6.1",
@@ -1994,7 +1951,6 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
"integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
- "dev": true,
"requires": {
"emoji-regex": "^7.0.1",
"is-fullwidth-code-point": "^2.0.0",
@@ -2014,7 +1970,6 @@
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
"integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
- "dev": true,
"requires": {
"ansi-regex": "^4.1.0"
}
@@ -2022,8 +1977,7 @@
"strip-eof": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
- "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
- "dev": true
+ "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
},
"strip-json-comments": {
"version": "2.0.1",
@@ -2195,7 +2149,6 @@
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
- "dev": true,
"requires": {
"isexe": "^2.0.0"
}
@@ -2203,8 +2156,7 @@
"which-module": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
- "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=",
- "dev": true
+ "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
},
"wide-align": {
"version": "1.1.3",
@@ -2258,7 +2210,6 @@
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz",
"integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==",
- "dev": true,
"requires": {
"ansi-styles": "^3.2.0",
"string-width": "^3.0.0",
@@ -2268,8 +2219,7 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
- "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
- "dev": true
+ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "5.2.2",
@@ -2316,14 +2266,12 @@
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
- "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
- "dev": true
+ "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
},
"yargs": {
"version": "13.2.4",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-13.2.4.tgz",
"integrity": "sha512-HG/DWAJa1PAnHT9JAhNa8AbAv3FPaiLzioSjCcmuXXhP8MlpHO5vwls4g4j6n30Z74GVQj8Xa62dWVx1QCGklg==",
- "dev": true,
"requires": {
"cliui": "^5.0.0",
"find-up": "^3.0.0",
@@ -2342,7 +2290,6 @@
"version": "13.1.0",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.0.tgz",
"integrity": "sha512-Yq+32PrijHRri0vVKQEm+ys8mbqWjLiwQkMFNXEENutzLPP0bE4Lcd4iA3OQY5HF+GD3xXxf0MEHb8E4/SA3AA==",
- "dev": true,
"requires": {
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
diff --git a/package.json b/package.json
index 81c2562..5612919 100644
--- a/package.json
+++ b/package.json
@@ -15,7 +15,8 @@
"dependencies": {
"axios": "^0.18.0",
"fs-extra": "^8.0.1",
- "lodash": "^4.17.11"
+ "lodash": "^4.17.11",
+ "yargs": "^13.2.4"
},
"devDependencies": {
"datacrate": "git+https://code.research.uts.edu.au/eresearch/datacrate.git#random_as_library",
diff --git a/test-data/fields.json b/test-data/fields.json
index 189e9e6..e9a82e3 100644
--- a/test-data/fields.json
+++ b/test-data/fields.json
@@ -45,6 +45,7 @@
}
},
"Person": {
+ "reverse": "in the reverse jsonld pointer",
"core": "dataset",
"format": "jsonld",
"recordTypeName": "person",
diff --git a/test/catalog.spec.js b/test/catalog.spec.js
index 11ce747..5072079 100644
--- a/test/catalog.spec.js
+++ b/test/catalog.spec.js
@@ -33,17 +33,11 @@ describe('catalog', function () {
let catalog = {};
before(function () {
- catalog = new CatalogSolr(catalogjson);
+ catalog = new CatalogSolr();
});
- describe('catalog', function () {
- it('should load catalog', function () {
- assert.strictEqual(_.isObject(catalog.jsonld['@context']), true, 'catalog not loaded')
- });
- });
-
- describe('load special fields', function () {
- it('should get special fields', function () {
+ describe('load config fields', function () {
+ it('should get config fields', function () {
const fields = require(fieldsPath);
const isConfig = catalog.setConfig(fields);
assert.strictEqual(isConfig, true, 'Config not complete');
@@ -58,7 +52,7 @@ describe('catalog', function () {
const fieldConfig = catalog.config;
- //TODO: Peter's idea is to convert everything into an array then it is safer to work to convert
+ //Peter's idea is to convert everything into an array then it is safer to work to convert
const graph = _.each(ca['@graph'], (g) => {
return catalog.ensureObjArray(g);
});
--
GitLab