Skip to content
Snippets Groups Projects
Commit b5343c68 authored by Moises Sacal's avatar Moises Sacal
Browse files

cleanup for using reduce instead to resolve promises withing the scope

parent 27344e6e
Branches
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.idea/workspace.xml
+ 31
− 26
View file @ b5343c68
......@@ -2,11 +2,7 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/generate-datacrates.js" beforeDir="false" afterPath="$PROJECT_DIR$/generate-datacrates.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/.tmp/" />
......@@ -32,8 +28,17 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="452">
<caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" />
<state relative-caret-position="125">
<caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state>
</provider>
</entry>
......@@ -47,11 +52,11 @@
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="606">
<caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" />
<state relative-caret-position="112">
<caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
</state>
</provider>
</entry>
......@@ -83,10 +88,10 @@
<option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
<option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/package.json" />
<option value="$PROJECT_DIR$/generate-datacrates.js" />
<option value="$PROJECT_DIR$/config.json" />
<option value="$PROJECT_DIR$/commit-to-solr.js" />
<option value="$PROJECT_DIR$/package.json" />
</list>
</option>
</component>
......@@ -195,7 +200,7 @@
<option name="presentableId" value="Default" />
<updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" />
<workItem from="1558654598476" duration="76397000" />
<workItem from="1558654598476" duration="77898000" />
</task>
<servers />
</component>
......@@ -232,19 +237,19 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="106875000" />
<option name="totallyTimeSpent" value="108376000" />
</component>
<component name="ToolWindowManager">
<frame x="1441" y="-277" width="1920" height="1177" extended-state="6" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.15654951" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32866243" />
<window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.28847927" />
<window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.3281106" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.28847927" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
......@@ -287,13 +292,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="330">
<caret line="22" selection-start-line="22" selection-end-line="22" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1761">
......@@ -352,15 +350,22 @@
</entry>
<entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="452">
<caret line="131" column="49" selection-start-line="131" selection-start-column="34" selection-end-line="131" selection-end-column="49" />
<state relative-caret-position="125">
<caret line="37" lean-forward="true" selection-start-line="37" selection-end-line="37" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="606">
<caret line="184" column="16" lean-forward="true" selection-start-line="184" selection-start-column="16" selection-end-line="184" selection-end-column="16" />
<state relative-caret-position="112">
<caret line="138" column="8" lean-forward="true" selection-start-line="138" selection-start-column="8" selection-end-line="138" selection-end-column="8" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="25" selection-start-line="20" selection-start-column="25" selection-end-line="20" selection-end-column="25" />
</state>
</provider>
</entry>
......
commit-to-solr.js
+ 23
− 55
View file @ b5343c68
......@@ -131,56 +131,7 @@ function catalogToArray(recs) {
return catalogs;
}
function batchIt(b) {
b.map(async (p, index) => {
try {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
records = entries(sourcePath, p);
catalogs = catalogToArray(records);
records = null;
let update = await updateDocs(solrUpdate, catalogs);
catalogs = null;
p = null;
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
} catch (e) {
console.log(e);
}
});
commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
console.log('solr commit');
return Promise.resolve();
}).catch((err) => {
return Promise.reject(err);
});
}
function reduceIt(b){
b.reduce((promise, p, index) => {
return promise.then(() => {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
const records = entries(sourcePath, p);
const catalogs = catalogToArray(records);
return updateDocs(solrUpdate, catalogs).then(async () => {
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (index >= b.length - 1) {
console.log('run commit');
return commitDocs(solrUpdate,'?commit=true&overwrite=true').then(() => {
return Promise.resolve();
});
}
return Promise.resolve();
});
}).catch((e) => {
console.log(e);
})
}, Promise.resolve());
}
// MAIN APP
let dirs = null;
if (fs.existsSync(sourcePath)) {
......@@ -192,11 +143,28 @@ if (fs.existsSync(sourcePath)) {
const batch = _.chunk(dirs, batchNum);
dirs = null;
let records = [];
let catalogs = [];
//batchIt(batch);
reduceIt(batch);
batch.reduce((promise, p, index) => {
return promise.then(() => {
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
const records = entries(sourcePath, p);
const catalogs = catalogToArray(records);
return updateDocs(solrUpdate, catalogs).then(async () => {
if (waitPeriod) {
const waited = await sleep(waitPeriod);
}
console.log(`batch ${index} of ${batch.length} : Update docs`);
if (index >= batch.length - 1) {
console.log('run commit');
return commitDocs(solrUpdate, '?commit=true&overwrite=true').then(() => {
return Promise.resolve();
});
}
return Promise.resolve();
});
}).catch((e) => {
console.log(e);
})
}, Promise.resolve());
if (logLevel >= 4) console.log(`Using: ${Math.round(process.memoryUsage().rss / 1024 / 1024 * 100) / 100} MBs`);
package.json
+ 0
− 2
View file @ b5343c68
......@@ -19,8 +19,6 @@
"datacrate": "git+https://code.research.uts.edu.au/eresearch/datacrate.git#random_as_library",
"fs-extra": "^8.0.1",
"lodash": "^4.17.11",
"request": "^2.88.0",
"request-promise": "^4.2.4",
"yargs": "^13.2.4"
},
"devDependencies": {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment