From a7ca257481747f03c111a3bb464ebcace8e96702 Mon Sep 17 00:00:00 2001
From: Moises Sacal <moisbo@gmail.com>
Date: Thu, 30 May 2019 10:20:10 +1000
Subject: [PATCH] added random generator for test
---
.gitignore | 3 +-
.idea/workspace.xml | 161 +++++++++++++++++++++--------------------
commit-to-solr.js | 2 +-
generate-datacrates.js | 42 +++++++++++
package.json | 4 +-
5 files changed, 131 insertions(+), 81 deletions(-)
create mode 100644 generate-datacrates.js
diff --git a/.gitignore b/.gitignore
index b940b81..04acb42 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
.idea
node_modules
-test-data/datacrates/
\ No newline at end of file
+test-data/datacrates/
+test-data/generated/
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index b990a90..017fffa 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -2,8 +2,11 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="ad75bb9b-12db-4cad-af64-4c68cecdb87b" name="Default Changelist" comment="">
+ <change afterPath="$PROJECT_DIR$/generate-datacrates.js" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/commit-to-solr.js" beforeDir="false" afterPath="$PROJECT_DIR$/commit-to-solr.js" afterDir="false" />
- <change beforePath="$PROJECT_DIR$/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config.json" afterDir="false" />
+ <change beforePath="$PROJECT_DIR$/package.json" beforeDir="false" afterPath="$PROJECT_DIR$/package.json" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/.tmp/" />
<ignored path="$PROJECT_DIR$/temp/" />
@@ -21,53 +24,26 @@
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="395">
- <caret line="121" column="6" selection-start-line="121" selection-start-column="6" selection-end-line="121" selection-end-column="6" />
- </state>
- </provider>
- </entry>
- </file>
- <file pinned="false" current-in-tab="false">
- <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
- <provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="300">
- <caret line="20" column="34" lean-forward="true" selection-start-line="20" selection-start-column="34" selection-end-line="20" selection-end-column="34" />
+ <state>
+ <caret column="20" selection-end-line="1" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
- <entry file="file://$PROJECT_DIR$/config.json">
- <provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="105">
- <caret line="7" column="15" lean-forward="true" selection-start-line="7" selection-start-column="15" selection-end-line="7" selection-end-column="15" />
- </state>
- </provider>
- </entry>
- </file>
- <file pinned="false" current-in-tab="false">
- <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
+ <entry file="file://$PROJECT_DIR$/generate-datacrates.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="394">
- <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
- </state>
- </provider>
- </entry>
- </file>
- <file pinned="false" current-in-tab="false">
- <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_3.js">
- <provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="1620">
- <caret line="108" column="29" selection-start-line="108" selection-start-column="29" selection-end-line="108" selection-end-column="29" />
+ <state relative-caret-position="45">
+ <caret line="3" column="31" selection-start-line="3" selection-start-column="31" selection-end-line="3" selection-end-column="31" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
- <entry file="file://$PROJECT_DIR$/package.json">
+ <entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="165">
- <caret line="11" column="4" selection-start-line="11" selection-start-column="4" selection-end-line="11" selection-end-column="4" />
+ <state relative-caret-position="90">
+ <caret line="6" column="27" lean-forward="true" selection-start-line="6" selection-start-column="27" selection-end-line="6" selection-end-column="27" />
</state>
</provider>
</entry>
@@ -130,19 +106,20 @@
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/lib/Catalog.js" />
- <option value="$PROJECT_DIR$/package.json" />
<option value="$PROJECT_DIR$/test-data/recordTypes.json" />
<option value="$PROJECT_DIR$/test-data/CATALOG.json" />
<option value="$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_3.js" />
<option value="$PROJECT_DIR$/test-data/CatalogSolr.json" />
- <option value="$PROJECT_DIR$/.gitignore" />
<option value="$PROJECT_DIR$/fields.json" />
<option value="$PROJECT_DIR$/test/catalog.spec.js" />
<option value="$PROJECT_DIR$/test-data/fields.json" />
<option value="$PROJECT_DIR$/lib/CatalogSolr.js" />
- <option value="$PROJECT_DIR$/commit-to-solr.js" />
<option value="$PROJECT_DIR$/config.json" />
+ <option value="$PROJECT_DIR$/commit-to-solr.js" />
+ <option value="$PROJECT_DIR$/package.json" />
+ <option value="$PROJECT_DIR$/.gitignore" />
+ <option value="$PROJECT_DIR$/generate-datacrates.js" />
</list>
</option>
</component>
@@ -170,6 +147,11 @@
<item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
<item name="lib" type="462c0819:PsiDirectoryNode" />
</path>
+ <path>
+ <item name="solr-catalog" type="b2602c69:ProjectViewProjectNode" />
+ <item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
+ <item name="test" type="462c0819:PsiDirectoryNode" />
+ </path>
<path>
<item name="solr-catalog" type="b2602c69:ProjectViewProjectNode" />
<item name="solr-catalog" type="462c0819:PsiDirectoryNode" />
@@ -215,10 +197,13 @@
</list>
</option>
</component>
- <component name="RunManager" selected="Node.js.commit-to-solr">
+ <component name="RunManager" selected="Node.js.generate-datacrates">
<configuration name="commit-to-solr" type="NodeJSConfigurationType" path-to-js-file="commit-to-solr.js" working-dir="$PROJECT_DIR$">
<method v="2" />
</configuration>
+ <configuration name="generate-datacrates" type="NodeJSConfigurationType" application-parameters="--d ./test-data/generated/ --n 10" path-to-js-file="generate-datacrates.js" working-dir="$PROJECT_DIR$">
+ <method v="2" />
+ </configuration>
<configuration name="catalog.spec.js" type="mocha-javascript-test-runner">
<node-interpreter>project</node-interpreter>
<node-options />
@@ -234,6 +219,7 @@
<list>
<item itemvalue="Mocha.catalog.spec.js" />
<item itemvalue="Node.js.commit-to-solr" />
+ <item itemvalue="Node.js.generate-datacrates" />
</list>
</component>
<component name="SvnConfiguration">
@@ -247,7 +233,7 @@
<option name="presentableId" value="Default" />
<updated>1558421147646</updated>
<workItem from="1558421149207" duration="30478000" />
- <workItem from="1558654598476" duration="51282000" />
+ <workItem from="1558654598476" duration="53066000" />
</task>
<servers />
</component>
@@ -284,7 +270,7 @@
</history-entry>
</component>
<component name="TimeTrackingManager">
- <option name="totallyTimeSpent" value="81760000" />
+ <option name="totallyTimeSpent" value="83544000" />
</component>
<component name="ToolWindowManager">
<frame x="1441" y="-277" width="1920" height="1177" extended-state="6" />
@@ -295,8 +281,8 @@
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
- <window_info active="true" anchor="bottom" id="Run" order="2" visible="true" weight="0.3299539" />
- <window_info anchor="bottom" id="Debug" order="3" weight="0.27834103" />
+ <window_info anchor="bottom" id="Run" order="2" weight="0.3299539" />
+ <window_info active="true" anchor="bottom" id="Debug" order="3" visible="true" weight="0.27834103" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
@@ -312,93 +298,112 @@
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
+ <component name="XDebuggerManager">
+ <breakpoint-manager>
+ <breakpoints>
+ <line-breakpoint enabled="true" type="javascript">
+ <url>file://$PROJECT_DIR$/generate-datacrates.js</url>
+ <line>13</line>
+ <properties lambdaOrdinal="-1" />
+ <option name="timeStamp" value="116" />
+ </line-breakpoint>
+ </breakpoints>
+ </breakpoint-manager>
+ </component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/test-data/datacrates/c1517eee-9bef-4d05-912c-6b73413718c7/CATALOG.json" />
<entry file="file://$PROJECT_DIR$/test-data/datacrates/4b4bc2b5-4a13-4369-be01-45ddceb440f3/CATALOG.json" />
<entry file="file://$PROJECT_DIR$/test-data/datacrates/43756ed8-b791-4507-a275-5bf8348de9f6/CATALOG.json" />
<entry file="file://$PROJECT_DIR$/test-data/datacrates/a564801a-eaf2-4461-8214-405151401157/CATALOG.json" />
<entry file="file://$PROJECT_DIR$/test-data/datacrates/c5192cd5-c8f4-4393-8bd1-f803ca944bac/CATALOG.json" />
- <entry file="file://$PROJECT_DIR$/package.json">
+ <entry file="file://$PROJECT_DIR$/fields.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="165">
- <caret line="11" column="4" selection-start-line="11" selection-start-column="4" selection-end-line="11" selection-end-column="4" />
+ <state relative-caret-position="164">
+ <caret line="29" column="20" lean-forward="true" selection-start-line="29" selection-start-column="20" selection-end-line="29" selection-end-column="20" />
</state>
</provider>
</entry>
- <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_3.js">
+ <entry file="file://$PROJECT_DIR$/test-data/CatalogSolr.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="1620">
- <caret line="108" column="29" selection-start-line="108" selection-start-column="29" selection-end-line="108" selection-end-column="29" />
+ <state relative-caret-position="-17">
+ <caret line="27" column="31" selection-start-line="27" selection-start-column="5" selection-end-line="27" selection-end-column="31" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/.gitignore">
+ <entry file="file://$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="30">
- <caret line="2" column="21" lean-forward="true" selection-start-line="2" selection-start-column="21" selection-end-line="2" selection-end-column="21" />
+ <state relative-caret-position="915">
+ <caret line="61" column="20" selection-start-line="61" selection-start-column="20" selection-end-line="61" selection-end-column="20" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/fields.json">
+ <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="164">
- <caret line="29" column="20" lean-forward="true" selection-start-line="29" selection-start-column="20" selection-end-line="29" selection-end-column="20" />
+ <state relative-caret-position="329">
+ <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/CatalogSolr.json">
+ <entry file="file://$PROJECT_DIR$/test-data/fields.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="-17">
- <caret line="27" column="31" selection-start-line="27" selection-start-column="5" selection-end-line="27" selection-end-column="31" />
+ <state relative-caret-position="45">
+ <caret line="3" column="23" lean-forward="true" selection-start-line="3" selection-start-column="23" selection-end-line="3" selection-end-column="23" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/FARMTOFREEWAYS_CATALOG.json">
+ <entry file="file://$PROJECT_DIR$/config.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="915">
- <caret line="61" column="20" selection-start-line="61" selection-start-column="20" selection-end-line="61" selection-end-column="20" />
+ <state relative-caret-position="135">
+ <caret line="9" lean-forward="true" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/test/catalog.spec.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="395">
- <caret line="121" column="6" selection-start-line="121" selection-start-column="6" selection-end-line="121" selection-end-column="6" />
+ <state>
+ <caret column="20" selection-end-line="1" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/CATALOG.json">
+ <entry file="file://$PROJECT_DIR$/.gitignore">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="329">
- <caret line="45" column="9" lean-forward="true" selection-start-line="45" selection-start-column="9" selection-end-line="45" selection-end-column="9" />
+ <state relative-caret-position="45">
+ <caret line="3" column="20" selection-start-line="3" selection-start-column="20" selection-end-line="3" selection-end-column="20" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
+ <entry file="file://$PROJECT_DIR$/package.json">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="394">
- <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
+ <state relative-caret-position="150">
+ <caret line="10" column="15" lean-forward="true" selection-start-line="10" selection-start-column="15" selection-end-line="10" selection-end-column="15" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/test-data/fields.json">
+ <entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_3.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="45">
- <caret line="3" column="23" lean-forward="true" selection-start-line="3" selection-start-column="23" selection-end-line="3" selection-end-column="23" />
+ <state relative-caret-position="1620">
+ <caret line="108" column="29" selection-start-line="108" selection-start-column="29" selection-end-line="108" selection-end-column="29" />
+ </state>
+ </provider>
+ </entry>
+ <entry file="file://$PROJECT_DIR$/lib/CatalogSolr.js">
+ <provider selected="true" editor-type-id="text-editor">
+ <state relative-caret-position="394">
+ <caret line="38" column="14" selection-start-line="38" selection-start-column="14" selection-end-line="38" selection-end-column="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/commit-to-solr.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="300">
- <caret line="20" column="34" lean-forward="true" selection-start-line="20" selection-start-column="34" selection-end-line="20" selection-end-column="34" />
+ <state relative-caret-position="90">
+ <caret line="6" column="27" lean-forward="true" selection-start-line="6" selection-start-column="27" selection-end-line="6" selection-end-column="27" />
</state>
</provider>
</entry>
- <entry file="file://$PROJECT_DIR$/config.json">
+ <entry file="file://$PROJECT_DIR$/generate-datacrates.js">
<provider selected="true" editor-type-id="text-editor">
- <state relative-caret-position="105">
- <caret line="7" column="15" lean-forward="true" selection-start-line="7" selection-start-column="15" selection-end-line="7" selection-end-column="15" />
+ <state relative-caret-position="45">
+ <caret line="3" column="31" selection-start-line="3" selection-start-column="31" selection-end-line="3" selection-end-column="31" />
</state>
</provider>
</entry>
diff --git a/commit-to-solr.js b/commit-to-solr.js
index e532daa..c6e7874 100644
--- a/commit-to-solr.js
+++ b/commit-to-solr.js
@@ -11,7 +11,7 @@ if (!fs.existsSync(configPath)) {
process.exit(1);
}
const configJson = require(configPath);
-const sourcePath = _.endsWith(configJson.source, '/') ? configJson.source : `${configJson.source}/`;
+const sourcePath = _.endsWith(configJson['source'], '/') ? configJson['source'] : `${configJson['source']}/`;
const solrUpdate = configJson['solrUpdate'] || '';
const fieldConfig = require(configJson['fields']);
const logLevel = configJson['logLevel'] || 4;
diff --git a/generate-datacrates.js b/generate-datacrates.js
new file mode 100644
index 0000000..6485fe0
--- /dev/null
+++ b/generate-datacrates.js
@@ -0,0 +1,42 @@
+const yargs = require('yargs');
+const randomize = require('datacrate/lib/randomize');
+const assert = require('assert');
+const fs = require('fs-extra');
+
+const argv = yargs['argv'];
+
+assert.notStrictEqual(argv.n, undefined, 'Please include --n as number of datacrates to generate');
+const numberOfDatacrates = argv.n;
+
+assert.notStrictEqual(argv.d, undefined, 'Please include --d as directory where datacrates are to be generated');
+const datacrateDirPath = argv.d;
+
+async function createDatacrates(dest, n) {
+ const sourcedata = await randomize.loadsourcedata('./node_modules/datacrate/vocabularies');
+ const datapubs = randomize.randdatapubs(n, sourcedata);
+ datapubs.reduce((promise, p, index) => {
+ return promise.then(async () => {
+ const id = await randomize.makedir(dest);
+ return randomize.makedatacrate(dest, p, id)
+ .then(() => {
+ if (index >= n) {
+ console.log("Done");
+ }
+ return Promise.resolve();
+ });
+ });
+ }, Promise.resolve());
+}
+
+try {
+ fs.ensureDirSync(datacrateDirPath);
+ createDatacrates(datacrateDirPath, numberOfDatacrates)
+ .then((res) => {
+ console.log(res);
+ })
+ .catch((err) => {
+ throw new Error(err);
+ })
+} catch (e) {
+ console.error(e.message);
+}
diff --git a/package.json b/package.json
index 5612919..d08bd2c 100644
--- a/package.json
+++ b/package.json
@@ -4,7 +4,9 @@
"description": "pushes catalog.json to solr via HTTP json",
"main": "index.js",
"scripts": {
- "test": "./node_modules/.bin/_mocha"
+ "test": "./node_modules/.bin/_mocha",
+ "commit": "node ./commit-to-solr.js --config ./config.json",
+ "generate": "node generate-datacrates.js -d ./test-data/generated/ -n 100"
},
"keywords": [
"solr",
--
GitLab