Can now crosswalk all of the sample ro-crates without dying

commit-to-solr.js

@@ -2,6 +2,7 @@ const axios = require('axios');
 const _ = require('lodash');
 const yargs = require('yargs');
 const CatalogSolr = require('./lib/CatalogSolr');
+const ROCrate = require('ro-crate').ROCrate;
 const fs = require('fs-extra');
 const path = require('path');
 const OCFLRepository = require('ocfl').Repository;
@@ -81,7 +82,7 @@ function solrObjects(recs) {
   const catalogs = [];
   recs.forEach((record) => {
     try {
-      const solrObj = catalog.createSolrObject(record, '@graph');
+      const solrObj = catalog.createSolrDocument(record);
       if (solrObj) {
         if (solrObj.Dataset) {
           solrObj.Dataset.forEach((c) => {
@@ -97,7 +98,7 @@ function solrObjects(recs) {
     } catch(e) {
       console.log("Error converting ro-crate to solr");
       console.log(e);
-      console.log("ro-crate id: " + get_ro_id(record));
+      console.log(JSON.stringify(record).substr(0, 160));
     }
 
   });
@@ -106,16 +107,6 @@ function solrObjects(recs) {
 }
 
 
-function get_ro_id(jsonld) {
-  const graph = jsonld['@graph'];
-  const root = graph.filter( item => {return item['path'] && item['path'].length > 0 && item['path'][0] === './'});
-  if( root.length < 1 ) {
-    return "[couldn't find path = ./]";
-  }
-
-  return root[0]['@id'];
-}
-
 
 async function loadFromDirs(root) {
   const e = await fs.stat(root);

config.json

 {
-  "source": "/Users/mike/working/redbox/ocfl-nginx/test_repos/rocrate_ocfl",
+  "source": "/Users/mike/working/ocfl-nginx/test_repos/rocrate_ocfl",
   "ocfl": 1,
   "dry-run": 1,
   "catalogFilename": "ro-crate-metadata.jsonld",

fields.json

 {
-  "Dataset": {
-    "filter": [
-      { "match_exact": [ "path", "./" ] }
-    ],
-    "core": "dataset",
-    "format": "jsonld",
-    "recordTypeName": "dataset",
-    "skip": ["hasPart"],
-    "flatten": {
-      "creator": {
-        "obj": "array"
+  "map_all": {
+    "@id": [ "id", "id_orig" ],
+    "@type": [ "record_type_s", "type_label" ]
+  },
+
+  "types": {
+
+    "Dataset": {
+      "path": {
+        "filter": { "re": "^\\./|data/$" }
       },
-      "GeoCoordinates": {}
-    },
-    "facets": {
-      "datePublished": {
-        "field_suffix": "_Dataset_facet",
-        "trim": true
+
+      "hasPart": {"skip": true},
+
+      "creator": {
+        "resolve": "multi",
+        "facet": true
       },
       "keywords": {
-        "trim": true,
-        "field_suffix": "_Dataset_facetmulti",
-        "tokenize": {
-          "delim": ","
+        "facet": {
+          "tokenize": ","
         }
       },
-      "creator": {
-        "trim": true,
-        "fieldName": "name",
-        "field_suffix": "_Dataset_facetmulti"
+      "datePublished": {
+        "facet": true
       },
       "publisher": {
-        "trim": true,
-        "fieldName": "@id",
-        "field_suffix": "_Dataset_facet"
-      },
-      "type": {
-        "trim": true,
-        "field_suffix": "facetmulti",
-        "skip_entry_type_suffix": true,
-        "escape_value": "solr_field"
+        "resolve": "single",
+        "facet": {
+          "field": "@id"
+        }
       }
     },
-    "oai-pmh": {
-      "core": "dataset",
-      "set": "datasource:ReDBox"
-    }
-  },
-  "Person": {
-    "core": "dataset",
-    "format": "jsonld",
-    "recordTypeName": "person",
-    "flatten": {
+
+    "Person": {
+
       "affiliation": {
-        "obj": "array"
-      }
-    },
-    "facets": {
-      "type": {
-        "trim": true,
-        "field_suffix": "_Dataset_facetmulti",
-        "skip_entry_type_suffix": true,
-        "escape_value": "solr_field"
+        "resolve": "multi"
       }
+
     }
   }
 }
\ No newline at end of file

lib/CatalogSolr.js

@@ -87,6 +87,10 @@ class CatalogSolr {
     const crate = new ROCrate(jsonld);
     crate.index();
 
+    this.rootId = this.getRootId(crate);
+
+    console.log(`Converting ${this.rootId}`);
+
     const cfMap = this.config['map_all'];
     const cfTypes = this.config['types'];
     const solrDocument = {};
@@ -144,8 +148,16 @@ class CatalogSolr {
 
 
   resolveValues(crate, cf, value) {
+    if( typeof value !== 'object' ) {
+      const error = this.convertError(`Can't resolve '${value}'`);
+      return ( cf === 'multi' ) ? [ error ] : error;
+    }
     if( cf === 'multi' ) {
-      return value.map((v) => this.resolveAndFlatten(crate, v));
+      if( Array.isArray(value) ) {
+        return value.map((v) => this.resolveAndFlatten(crate, v));
+      } else {
+        return [ this.resolveAndFlatten(crate, value) ];
+      }
     } else {
       if( Array.isArray(value) ) {
         return this.resolveAndFlatten(crate, value[0]);
@@ -201,8 +213,8 @@ class CatalogSolr {
   // TODO - this should give better context
 
   convertError(message) {
-    const wrapped = `Conversion error: ${message}`;
-    console.log(wrapped);
+    const wrapped = `[ERROR]: ${message}`;
+    console.log(this.rootId + ' ' + wrapped);
     return wrapped;
   }
 
@@ -233,6 +245,20 @@ class CatalogSolr {
     }
   }
 
+  getRootId(crate) {
+    const root = crate.graph.filter((item) => {
+      return item['path'] && ( item['path'] === './' || item['path'] === './data');
+    });
+    if( root.length < 1 ) {
+      console.log("Couldn't find root item by path");
+      return undefined;
+    }
+    if( root.length > 1 ) {
+      console.log("Multiple matches for root path");
+    }
+    return root[0]['@id'];
+  }
+
 
 }