Skip to content
Snippets Groups Projects
Commit b5343c68 authored by Moises Sacal's avatar Moises Sacal
Browse files

cleanup for using reduce instead to resolve promises withing the scope

parent 27344e6e
Branches
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.idea/workspace.xml
+ 31
− 26
View file @ b5343c68
...@@ -2,11 +2,7 @@ ...@@ -2,11 +2,7 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment=""> <list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" /> <change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/generate-datacrates.js" beforeDir="false" afterPath="$PROJECT_DIR$/generate-datacrates.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" /> <change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
</list> </list>
<ignored path="$PROJECT_DIR$/.tmp/" /> <ignored path="$PROJECT_DIR$/.tmp/" />
...@@ -32,8 +28,17 @@ ...@@ -32,8 +28,17 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js"> <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="452"> <state relative-caret-position="125">
<caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" /> <caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -47,11 +52,11 @@ ...@@ -47,11 +52,11 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/commit-to-solr.js"> <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="606"> <state relative-caret-position="112">
<caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" /> <caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
</state> </state>
</provider> </provider>
</entry> </entry>
...@@ -83,10 +88,10 @@ ...@@ -83,10 +88,10 @@
<option value="$PROJECT_DIR$/test-data/fields.json" /> <option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/lib/CatalogSolr.js" /> <option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
<option value="$PROJECT_DIR$/.gitignore" /> <option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/package.json" />
<option value="$PROJECT_DIR$/generate-datacrates.js" /> <option value="$PROJECT_DIR$/generate-datacrates.js" />
<option value="$PROJECT_DIR$/config.json" /> <option value="$PROJECT_DIR$/config.json" />
<option value="$PROJECT_DIR$/commit-to-solr.js" /> <option value="$PROJECT_DIR$/commit-to-solr.js" />
<option value="$PROJECT_DIR$/package.json" />
</list> </list>
</option> </option>
</component> </component>
...@@ -195,7 +200,7 @@ ...@@ -195,7 +200,7 @@
<option name="presentableId" value="Default" /> <option name="presentableId" value="Default" />
<updated>1558421147646</updated> <updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" /> <workItem from="1558421149207" duration="30478000" />
<workItem from="1558654598476" duration="76397000" /> <workItem from="1558654598476" duration="77898000" />
</task> </task>
<servers /> <servers />
</component> </component>
...@@ -232,19 +237,19 @@ ...@@ -232,19 +237,19 @@
</history-entry> </history-entry>
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="106875000" /> <option name="totallyTimeSpent" value="108376000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="1441" y="-277" width="1920" height="1177" extended-state="6" /> <frame x="1441" y="-277" width="1920" height="1177" extended-state="6" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" /> <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" /> <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" /> <window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" /> <window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" /> <window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32866243" /> <window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.3281106" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.28847927" /> <window_info anchor="bottom" id="Debug" order="3" weight="0.28847927" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" /> <window_info anchor="bottom" id="TODO" order="6" />
...@@ -287,13 +292,6 @@ ...@@ -287,13 +292,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="22" selection-start-line="22" selection-end-line="22" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js"> <entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1761"> <state relative-caret-position="-1761">
...@@ -352,15 +350,22 @@ ...@@ -352,15 +350,22 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js"> <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="452"> <state relative-caret-position="125">
<caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" /> <caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/commit-to-solr.js"> <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="606"> <state relative-caret-position="112">
<caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" /> <caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state> </state>
</provider> </provider>
</entry> </entry>
......
commit-to-solr.js
+ 23
− 55
View file @ b5343c68
...@@ -131,56 +131,7 @@ function catalogToArray(recs) { ...@@ -131,56 +131,7 @@ function catalogToArray(recs) {
return catalogs; return catalogs;
} }
function batchIt(b) { // MAIN APP
b.map(async (p, index) => {
try {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
records = entries(sourcePath, p);
catalogs = catalogToArray(records);
records = null;
let update = await updateDocs(solrUpdate, catalogs);
catalogs = null;
p = null;
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
} catch (e) {
console.log(e);
}
});
commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
console.log('solr commit');
return Promise.resolve();
}).catch((err) => {
return Promise.reject(err);
});
}
function reduceIt(b){
b.reduce((promise, p, index) => {
return promise.then(() => {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
const records = entries(sourcePath, p);
const catalogs = catalogToArray(records);
return updateDocs(solrUpdate, catalogs).then(async () => {
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (index >= b.length - 1) {
console.log('run commit');
return commitDocs(solrUpdate,'?commit=true&overwrite=true').then(() => {
return Promise.resolve();
});
}
return Promise.resolve();
});
}).catch((e) => {
console.log(e);
})
}, Promise.resolve());
}
let dirs = null; let dirs = null;
if (fs.existsSync(sourcePath)) { if (fs.existsSync(sourcePath)) {
...@@ -192,11 +143,28 @@ if (fs.existsSync(sourcePath)) { ...@@ -192,11 +143,28 @@ if (fs.existsSync(sourcePath)) {
const batch = _.chunk(dirs, batchNum); const batch = _.chunk(dirs, batchNum);
dirs = null; dirs = null;
let records = [];
let catalogs = [];
//batchIt(batch); batch.reduce((promise, p, index) => {
reduceIt(batch); return promise.then(() => {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
const records = entries(sourcePath, p);
const catalogs = catalogToArray(records);
return updateDocs(solrUpdate, catalogs).then(async () => {
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (index >= batch.length - 1) {
console.log('run commit');
return commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
return Promise.resolve();
});
}
return Promise.resolve();
});
}).catch((e) => {
console.log(e);
})
}, Promise.resolve());
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
package.json
+ 0
− 2
View file @ b5343c68
...@@ -19,8 +19,6 @@ ...@@ -19,8 +19,6 @@
"datacrate": "git+https://code.research.uts.edu.au/eresearch/datacrate.git#random_as_library", "datacrate": "git+https://code.research.uts.edu.au/eresearch/datacrate.git#random_as_library",
"fs-extra": "^8.0.1", "fs-extra": "^8.0.1",
"lodash": "^4.17.11", "lodash": "^4.17.11",
"request": "^2.88.0",
"request-promise": "^4.2.4",
"yargs": "^13.2.4" "yargs": "^13.2.4"
}, },
"devDependencies": { "devDependencies": {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment