diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 8e69ce401a4f92f6f4390a405516369bc5503176..b990a901df28d3a1ffdfe68ec3fbb80c7bc50ab7 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,15 +2,8 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
- <change afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
- <change afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
- <change afterPath="$PROJECT_DIR$/fields.json" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/lib/CatalogSolr.js" beforeDir="false" afterPath="$PROJECT_DIR$/lib/CatalogSolr.js" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/test-data/fields.json" beforeDir="false" afterPath="$PROJECT_DIR$/test-data/fields.json" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/test/catalog.spec.js" beforeDir="false" afterPath="$PROJECT_DIR$/test/catalog.spec.js" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/.tmp/" />
<ignored path="$PROJECT_DIR$/temp/" />
@@ -37,25 +30,25 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="587">
- <caret line="137" column="31" lean-forward="true" selection-start-line="137" selection-start-column="31" selection-end-line="137" selection-end-column="31" />
+ <state relative-caret-position="300">
+ <caret line="20" column="34" lean-forward="true" selection-start-line="20" selection-start-column="34" selection-end-line="20" selection-end-column="34" />
</state>
</provider>
</entry>
</file>
- <file pinned="false" current-in-tab="false">
+ <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="75">
- <caret line="5" column="18" selection-start-line="5" selection-start-column="18" selection-end-line="5" selection-end-column="18" />
+ <state relative-caret-position="105">
+ <caret line="7" column="15" lean-forward="true" selection-start-line="7" selection-start-column="15" selection-end-line="7" selection-end-column="15" />
</state>
</provider>
</entry>
</file>
- <file pinned="false" current-in-tab="true">
+ <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="245">
+ <state relative-caret-position="394">
<caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
</state>
</provider>
@@ -95,8 +88,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/test-data/fields.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="409">
- <caret line="32" column="39" selection-start-line="32" selection-start-column="25" selection-end-line="32" selection-end-column="39" />
+ <state relative-caret-position="45">
+ <caret line="3" column="23" lean-forward="true" selection-start-line="3" selection-start-column="23" selection-end-line="3" selection-end-column="23" />
</state>
</provider>
</entry>
@@ -145,16 +138,16 @@
<option value="$PROJECT_DIR$/test-data/CatalogSolr.json" />
<option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/fields.json" />
- <option value="$PROJECT_DIR$/config.json" />
- <option value="$PROJECT_DIR$/commit-to-solr.js" />
<option value="$PROJECT_DIR$/test/catalog.spec.js" />
<option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
+ <option value="$PROJECT_DIR$/commit-to-solr.js" />
+ <option value="$PROJECT_DIR$/config.json" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="6">
- <option name="x" value="1440" />
+ <option name="x" value="1441" />
<option name="y" value="-277" />
<option name="width" value="1920" />
<option name="height" value="1177" />
@@ -254,7 +247,7 @@
<option name="presentableId" value="Default" />
<updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" />
- <workItem from="1558654598476" duration="48948000" />
+ <workItem from="1558654598476" duration="51282000" />
</task>
<servers />
</component>
@@ -291,24 +284,25 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
- <option name="totallyTimeSpent" value="79426000" />
+ <option name="totallyTimeSpent" value="81760000" />
</component>
<component name="ToolWindowManager">
- <frame x="1440" y="-277" width="1920" height="1177" extended-state="6" />
+ <frame x="1441" y="-277" width="1920" height="1177" extended-state="6" />
+ <editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.15481833" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
- <window_info anchor="bottom" id="Run" order="2" weight="0.3299539" />
- <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.27834103" />
+ <window_info active="true" anchor="bottom" id="Run" order="2" visible="true" weight="0.3299539" />
+ <window_info anchor="bottom" id="Debug" order="3" weight="0.27834103" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="8" />
- <window_info anchor="bottom" id="Terminal" order="9" weight="0.23821656" />
+ <window_info anchor="bottom" id="Terminal" order="9" weight="0.23778802" />
<window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
<window_info anchor="right" id="Commander" order="0" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
@@ -352,13 +346,6 @@
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/config.json">
- <provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="75">
- <caret line="5" column="18" selection-start-line="5" selection-start-column="18" selection-end-line="5" selection-end-column="18" />
- </state>
- </provider>
- </entry>
<entry file="file://$PROJECT_DIR$/test-data/CatalogSolr.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-17">
@@ -380,31 +367,38 @@
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
+ <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="587">
- <caret line="137" column="31" lean-forward="true" selection-start-line="137" selection-start-column="31" selection-end-line="137" selection-end-column="31" />
+ <state relative-caret-position="329">
+ <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
+ <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="329">
- <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
+ <state relative-caret-position="394">
+ <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test-data/fields.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="409">
- <caret line="32" column="39" selection-start-line="32" selection-start-column="25" selection-end-line="32" selection-end-column="39" />
+ <state relative-caret-position="45">
+ <caret line="3" column="23" lean-forward="true" selection-start-line="3" selection-start-column="23" selection-end-line="3" selection-end-column="23" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
+ <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="245">
- <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
+ <state relative-caret-position="300">
+ <caret line="20" column="34" lean-forward="true" selection-start-line="20" selection-start-column="34" selection-end-line="20" selection-end-column="34" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/config.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="105">
+ <caret line="7" column="15" lean-forward="true" selection-start-line="7" selection-start-column="15" selection-end-line="7" selection-end-column="15" />
</state>
</provider>
</entry>
diff --git a/commit-to-solr.js b/commit-to-solr.js
index 7276d0103b358c589ca8be1c6430053d225f6113..e532daac370410c2cb5609db674db80a29beb827 100644
--- a/commit-to-solr.js
+++ b/commit-to-solr.js
@@ -16,6 +16,7 @@ const solrUpdate = configJson['solrUpdate'] || '';
const fieldConfig = require(configJson['fields']);
const logLevel = configJson['logLevel'] || 4;
const waitPeriod = configJson['waitPeriod'] || 0;
+const batchNum = configJson['batch'] || 1000;
const catalog = new CatalogSolr();
catalog.setConfig(fieldConfig);
@@ -101,35 +102,37 @@ if (fs.existsSync(sourcePath)) {
process.exit(1);
}
-catalogs = [];
-
-records.forEach((r) => {
- const solrObj = createCatalogSolr(r);
- if (solrObj) {
- if (solrObj.Dataset) {
- solrObj.Dataset.forEach((c) => {
- catalogs.push(c);
- });
- }
- if (solrObj.Person) {
- solrObj.Person.forEach((c) => {
- catalogs.push(c);
- });
+function catalogToArray(recs) {
+ const catalogs = [];
+ recs.forEach((rec) => {
+ const solrObj = createCatalogSolr(rec);
+ if (solrObj) {
+ if (solrObj.Dataset) {
+ solrObj.Dataset.forEach((c) => {
+ catalogs.push(c);
+ });
+ }
+ if (solrObj.Person) {
+ solrObj.Person.forEach((c) => {
+ catalogs.push(c);
+ });
+ }
}
- }
-
-});
+ });
+ return catalogs;
+}
-const fakePromises = [1];
+const batch = _.chunk(records, batchNum);
-fakePromises.reduce((promise, p, index) => {
+batch.reduce((promise, p, index) => {
return promise.then(() => {
+ const catalogs = catalogToArray(records);
return updateDocs(catalogs).then(async () => {
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
console.log('Update docs');
- if (index >= fakePromises.length - 1) {
+ if (index >= batch.length - 1) {
console.log('run commit');
return commitDocs('?commit=true&overwrite=true').then(() => {
return Promise.resolve();
diff --git a/config.json b/config.json
index 36c2652a348bfbd9ef2ce6fa52dfa2f1f0c19ae2..8bf2a719ae236d787c4b409cf56e7c60c6753a83 100644
--- a/config.json
+++ b/config.json
@@ -4,5 +4,6 @@
"solrUpdate": "http://localhost:8983/solr/dataset/update/json",
"logLevel": 3,
"waitPeriod": 1,
- "fields": "./fields.json"
+ "fields": "./fields.json",
+ "batch": 1000
}