diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 0ce6dd310f7029af638a871016fea3bf2fb2ab1c..8422d715231aae45c29565a32dc95aaad9b767fd 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,11 +2,7 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
- <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/generate-datacrates.js" beforeDir="false" afterPath="$PROJECT_DIR$/generate-datacrates.js" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/.tmp/" />
@@ -32,8 +28,17 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="452">
- <caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" />
+ <state relative-caret-position="125">
+ <caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
+ </state>
+ </provider>
+ </entry>
+ </file>
+ <file pinned="false" current-in-tab="true">
+ <entry file="file://$PROJECT_DIR$/package.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="300">
+ <caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state>
</provider>
</entry>
@@ -47,11 +52,11 @@
</provider>
</entry>
</file>
- <file pinned="false" current-in-tab="true">
+ <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="606">
- <caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" />
+ <state relative-caret-position="112">
+ <caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
</state>
</provider>
</entry>
@@ -83,10 +88,10 @@
<option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
<option value="$PROJECT_DIR$/.gitignore" />
- <option value="$PROJECT_DIR$/package.json" />
<option value="$PROJECT_DIR$/generate-datacrates.js" />
<option value="$PROJECT_DIR$/config.json" />
<option value="$PROJECT_DIR$/commit-to-solr.js" />
+ <option value="$PROJECT_DIR$/package.json" />
</list>
</option>
</component>
@@ -195,7 +200,7 @@
<option name="presentableId" value="Default" />
<updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" />
- <workItem from="1558654598476" duration="76397000" />
+ <workItem from="1558654598476" duration="77898000" />
</task>
<servers />
</component>
@@ -232,19 +237,19 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
- <option name="totallyTimeSpent" value="106875000" />
+ <option name="totallyTimeSpent" value="108376000" />
</component>
<component name="ToolWindowManager">
<frame x="1441" y="-277" width="1920" height="1177" extended-state="6" />
<editor active="true" />
<layout>
- <window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" />
+ <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
- <window_info anchor="bottom" id="Run" order="2" weight="0.32866243" />
- <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.28847927" />
+ <window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.3281106" />
+ <window_info anchor="bottom" id="Debug" order="3" weight="0.28847927" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
@@ -287,13 +292,6 @@
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/package.json">
- <provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="330">
- <caret line="22" selection-start-line="22" selection-end-line="22" />
- </state>
- </provider>
- </entry>
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1761">
@@ -352,15 +350,22 @@
</entry>
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="452">
- <caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" />
+ <state relative-caret-position="125">
+ <caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="606">
- <caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" />
+ <state relative-caret-position="112">
+ <caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/package.json">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="300">
+ <caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state>
</provider>
</entry>
diff --git a/commit-to-solr.js b/commit-to-solr.js
index 4aab2f5f2dd7782afba3106fae5c97774bf19928..8a3b6d5e4a4910c30a51d91e518eaa40f78cacf2 100644
--- a/commit-to-solr.js
+++ b/commit-to-solr.js
@@ -131,56 +131,7 @@ function catalogToArray(recs) {
return catalogs;
}
-function batchIt(b) {
- b.map(async (p, index) => {
- try {
- if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
- records = entries(sourcePath, p);
- catalogs = catalogToArray(records);
- records = null;
- let update = await updateDocs(solrUpdate, catalogs);
- catalogs = null;
- p = null;
- console.log(`batch ${index} of ${batch.length} : Update docs`);
- if (waitPeriod) {
- const waited = await sleep(waitPeriod);
- }
- } catch (e) {
- console.log(e);
- }
- });
- commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
- console.log('solr commit');
- return Promise.resolve();
- }).catch((err) => {
- return Promise.reject(err);
- });
-}
-
-function reduceIt(b){
- b.reduce((promise, p, index) => {
- return promise.then(() => {
- if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
- const records = entries(sourcePath, p);
- const catalogs = catalogToArray(records);
- return updateDocs(solrUpdate, catalogs).then(async () => {
- if (waitPeriod) {
- const waited = await sleep(waitPeriod);
- }
- console.log(`batch ${index} of ${batch.length} : Update docs`);
- if (index >= b.length - 1) {
- console.log('run commit');
- return commitDocs(solrUpdate,'?commit=true&overwrite=true').then(() => {
- return Promise.resolve();
- });
- }
- return Promise.resolve();
- });
- }).catch((e) => {
- console.log(e);
- })
- }, Promise.resolve());
-}
+// MAIN APP
let dirs = null;
if (fs.existsSync(sourcePath)) {
@@ -192,11 +143,28 @@ if (fs.existsSync(sourcePath)) {
const batch = _.chunk(dirs, batchNum);
dirs = null;
-let records = [];
-let catalogs = [];
-//batchIt(batch);
-reduceIt(batch);
+batch.reduce((promise, p, index) => {
+ return promise.then(() => {
+ if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
+ const records = entries(sourcePath, p);
+ const catalogs = catalogToArray(records);
+ return updateDocs(solrUpdate, catalogs).then(async () => {
+ if (waitPeriod) {
+ const waited = await sleep(waitPeriod);
+ }
+ console.log(`batch ${index} of ${batch.length} : Update docs`);
+ if (index >= batch.length - 1) {
+ console.log('run commit');
+ return commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
+ return Promise.resolve();
+ });
+ }
+ return Promise.resolve();
+ });
+ }).catch((e) => {
+ console.log(e);
+ })
+}, Promise.resolve());
-if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
diff --git a/package.json b/package.json
index 0c3dce3d588fee82bee8caca4b29e24483000582..8f4506677d8c41265dacfb74cd942c6cc8fa8fd2 100644
--- a/package.json
+++ b/package.json
@@ -19,8 +19,6 @@
"datacrate": "git+https://code.research.uts.edu.au/eresearch/datacrate.git#random_as_library",
"fs-extra": "^8.0.1",
"lodash": "^4.17.11",
- "request": "^2.88.0",
- "request-promise": "^4.2.4",
"yargs": "^13.2.4"
},
"devDependencies": {