Skip to content
Snippets Groups Projects
Commit 9134db14 authored by Mike Lynch's avatar Mike Lynch
Browse files

Sorted out a lot of bugs in the solr schema section. I think this should fix the problem 

where we'd get multiple entries for copy-fields
parent 0e9ee800
Branches
No related merge requests found
Hide whitespace changes
Inline Side-by-side
commit-to-solr.js
+ 61
− 17
View file @ 9134db14
......@@ -90,7 +90,7 @@ async function updateSchema(solrURL, schemaFile) {
for( const type of Object.keys(schemaConf) ) {
for( const field of schemaConf[type] ) {
console.log(`Setting schema field ${type} ${JSON.stringify(field)}`);
//console.log(`Setting schema field ${type} ${JSON.stringify(field)}`);
await setSchemaField(solrURL, type, field);
}
}
......@@ -98,25 +98,32 @@ async function updateSchema(solrURL, schemaFile) {
async function setSchemaField(solrURL, fieldtype, schemaJson) {
const url = solrURL + '/fields';
const url = solrURL + '/' + fieldtype + 's';
const schemaAPIJson = {};
if( fieldtype !== 'copy-field' ) {
const name = schemaJson['name'];
if( await schemaFieldExists(solrURL, name) ) {
console.log("replacing it");
schemaAPIJson['replace-' + fieldtype] = schemaJson;
const name = schemaJson['name'];
// solr copyfields are annoying because they don't have a name and
// can't be replaced, so I'm trying to delete them and ignoring errors.
if( fieldtype === 'copyfield' ) {
console.log(`Deleting copyfield ${JSON.stringify(schemaJson)}`);
await tryDeleteCopyField(solrURL, schemaJson);
schemaAPIJson['add-copy-field'] = schemaJson;
} else {
const apifield = ( fieldtype === 'field' ) ? 'field' : 'dynamic-field';
if( await schemaFieldExists(url, name) ) {
console.log(`Schema: replacing ${fieldtype} ${name}`);
schemaAPIJson['replace-' + apifield] = schemaJson;
} else {
console.log("adding it");
schemaAPIJson['add-' + fieldtype] = schemaJson;
console.log(`Schema: adding ${fieldtype} ${name}`);
schemaAPIJson['add-' + apifield] = schemaJson;
}
} else {
schemaAPIJson['add-copy-field'] = schemaJson;
}
try {
console.log(`Posting to schema API ${url} ${JSON.stringify(schemaAPIJson)}`);
const response = await axios({
url: url,
url: solrURL,
method: 'post',
data: schemaAPIJson,
responseType: 'json',
......@@ -124,14 +131,21 @@ async function setSchemaField(solrURL, fieldtype, schemaJson) {
'Content-Type': 'application/json; charset=utf-8'
}
});
console.log("Response: " + response.status);
//console.log("Response: " + response.status);
} catch(e) {
console.log("Error updating schema");
console.log(`URL: ${url}`);
console.log(`schemaAPIJson: ${JSON.stringify(schemaAPIJson)}`);
if( e.response ) {
console.log(`${e.response.status} ${e.response.statusText}`);
} else {
console.log(e);
}
}
}
async function schemaFieldExists(solrURL, field) {
const url = solrURL + '/fields/' + field;
const url = solrURL + '/' + field;
try {
const resp = await axios ({
url: url,
......@@ -152,6 +166,36 @@ async function schemaFieldExists(solrURL, field) {
}
}
async function tryDeleteCopyField(solrURL, copyFieldJson) {
try {
const resp = await axios ({
url: solrURL,
method: 'post',
data: { "delete-copy-field": { source: copyFieldJson['source'], dest: copyFieldJson['dest'] } },
responseType: 'json',
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
});
console.log("copyfield removed");
return true;
} catch(e) {
if( e.response ) {
if( e.response.status === 404 ) {
console.log("Schema field " + field + " not found");
return false;
} else {
console.log("copy field delete error " + e.response.status);
return false;
}
} else {
console.log("unknown error " + e);
throw(e);
return false;
}
}
}
async function purgeSolr() {
......@@ -226,10 +270,10 @@ function solrObjects(recs) {
const jsonld = record['jsonld'];
const docs = indexer.createSolrDocument(jsonld);
if (docs) {
console.log(`Got solr docs of type: ${Object.keys(docs)}`);
for (let t of Object.keys(docs)){
if (t === "Dataset") {
docs.Dataset.forEach((dataset) => {
console.log("Dataset (a) = " + JSON.stringify(dataset));
dataset['path'] = record['path'];
if( uriIds === 'hashpaths' ) {
dataset['uri_id'] = record['hash_path'];
......@@ -240,9 +284,7 @@ function solrObjects(recs) {
console.log("Warning: couldn't find id for uri_id");
}
};
console.log("Dataset (b) = " + JSON.stringify(dataset));
solrDocs.push(dataset);
console.log(`Dataset URI id ${dataset['uri_id']}`);
});
} else {
docs[t].forEach((item) => {
......@@ -336,6 +378,8 @@ async function main () {
await updateSchema(solrSchema, configJson['schema']);
}
return;
const records = await loadFromOcfl(sourcePath);
await commitBatches(records);
}
......
fields.json
+ 0
− 27
View file @ 9134db14
......@@ -85,33 +85,6 @@
"multi": true,
"inherit": true
}
},
"Organization": {
"@id" : {
"filter" : { "re" : "NOMATCH"}
},
"@reverse": { "skip": true },
"contactPoint" : {
"multi": "true",
"escapedJSON": true
},
"geo" : {
"multi": "true",
"escapedJSON": true
},
"address" : {
"multi": "true"
},
"description" : {
"multi": "true"
},
"license": {
"multi": true,
"inherit": true
}
}
}
}
\ No newline at end of file
lib/CatalogSolr.js
+ 4
− 6
View file @ 9134db14
......@@ -237,10 +237,10 @@ Types with errors: ${this.errors.join(', ')}`);
if (!(solrDocument[type])) {
solrDocument[type] = [];
}
if (type === "Organization") {
console.log("Doing an org", item);
console.log(solr);
}
// if (type === "Organization") {
// console.log("Doing an org", item);
// console.log(solr);
// }
solrDocument[type].push(solr);
}
......@@ -252,11 +252,9 @@ Types with errors: ${this.errors.join(', ')}`);
// map the fields in an an ro-crate item to a solr document
mapItem(cfBase, cf, type, item) {
console.log(`*** mapItem ${type} ${JSON.stringify(item)}`);
const solr = this.baseSolr(cfBase, item);
_.each(item, ( value, field ) => {
const fieldcf = cf[field];
console.log(`-> ${field} ${JSON.stringify(fieldcf)}`);
if( !(field in cfBase ) ) {
if( ! fieldcf ) {
// no config for this field so copy
......
schema.json
+ 3
− 3
View file @ 9134db14
......@@ -40,7 +40,7 @@
"multiValued": true
}
],
"dynamic-field": [
"dynamicfield": [
{
"name": "*_name",
"type": "text_en",
......@@ -89,7 +89,7 @@
"stored": true
}
],
"copy-field": [
"copyfield": [
{
"source": "name",
"dest": [
......@@ -97,7 +97,7 @@
]
},
{
"source": "datePublished",
"source": "*_datePublished",
"maxChars": 4,
"dest": [
"yearPublished"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment