Commit 2e325cfb authored by PTSEFTON's avatar PTSEFTON
Browse files

Made some changes to ID handling and then undid them.

parent 566949d2
...@@ -48,10 +48,8 @@ program ...@@ -48,10 +48,8 @@ program
"Create a zipped version of the bag - only applies with --bag" "Create a zipped version of the bag - only applies with --bag"
) )
.option("-d, --depth", "Maximum depth to recurse into directories") .option("-d, --depth", "Maximum depth to recurse into directories")
.option( .option("-r, --recurse", "Recurse into directories looking for CATALOG_.xslc files")
"-r, --recurse",
"Recurse into directories - up to " + defaults.max_depth
)
.option( .option(
"-u, --url [distro]", "-u, --url [distro]",
"Distribution URL" "Distribution URL"
...@@ -111,6 +109,7 @@ console.log("Generating html from exsiting " + catalog_path + " file"); ...@@ -111,6 +109,7 @@ console.log("Generating html from exsiting " + catalog_path + " file");
catalog_path = path.join(process.cwd(), catalog_path); catalog_path = path.join(process.cwd(), catalog_path);
} }
var dir = path.dirname(catalog_path) var dir = path.dirname(catalog_path)
shell.rm("-rf", path.join(dir, "CATALOG_files"));
var json_helper = new JSON_helper(); var json_helper = new JSON_helper();
json_helper.init(require(catalog_path)); json_helper.init(require(catalog_path));
if (program.url){ if (program.url){
......
This source diff could not be displayed because it is too large. You can view the blob instead.
...@@ -36,6 +36,7 @@ module.exports = function() { ...@@ -36,6 +36,7 @@ module.exports = function() {
defaults.DataCrate_Specification_Identifier defaults.DataCrate_Specification_Identifier
}; };
if (this.helper.root_node["contactPoint"] && this.helper.root_node["contactPoint"]["@id"]) { if (this.helper.root_node["contactPoint"] && this.helper.root_node["contactPoint"]["@id"]) {
contact = this.helper.item_by_id[this.helper.root_node["contactPoint"]["@id"]]; contact = this.helper.item_by_id[this.helper.root_node["contactPoint"]["@id"]];
map = { map = {
...@@ -80,17 +81,22 @@ module.exports = function() { ...@@ -80,17 +81,22 @@ module.exports = function() {
shell.exec("bagit updatetagmanifests " + this.target_dir); shell.exec("bagit updatetagmanifests " + this.target_dir);
}, },
fix_paths: function fix_paths(catalog_path) { fix_paths: function fix_paths(catalog_path) {
console.log(shell.test("-f", catalog_path));
if (!path.isAbsolute(catalog_path)) { if (!path.isAbsolute(catalog_path)) {
catalog_path = path.join("./", catalog_path); catalog_path = path.join("./", catalog_path);
} }
var catalog = this.helper.json_ld; var catalog = this.helper.json_ld;
for (let item of catalog["@graph"]) { for (let item of catalog["@graph"]) {
if (item["path"]) { if (item["path"]) {
var p = this.helper.value_as_array(item["path"])[0]; var p = this.helper.value_as_array(item["path"])[0];
var new_p = path.join("./data/", p); var new_p = path.join("data", p);
item["path"] = [new_p]; item["path"] = [new_p];
if (item["@id"] === p) {
this.helper.update_id(item, new_p)
}
} }
this.helper.update_all_ids();
} }
fs.writeFileSync( fs.writeFileSync(
catalog_path, catalog_path,
......
...@@ -23,6 +23,7 @@ var defaults = require('./defaults'); ...@@ -23,6 +23,7 @@ var defaults = require('./defaults');
var XLSX = require('xlsx'); var XLSX = require('xlsx');
var path = require('path'); var path = require('path');
const ignore= /^\./; const ignore= /^\./;
const querystring = require('querystring')
const Property = require("./property.js"); const Property = require("./property.js");
const Item = require("./item.js"); const Item = require("./item.js");
...@@ -65,13 +66,15 @@ module.exports = function() { ...@@ -65,13 +66,15 @@ module.exports = function() {
item_json[name_value["Name"]] = [name_value["Value"]]; item_json[name_value["Name"]] = [name_value["Value"]];
} }
} }
item_json["TYPE:"] = "Dataset"; item_json["TYPE:"] = "Dataset";
item_json["path"] = collection.rel_path; item_json["path"] = collection.rel_path;
if(!item_json["Name"]) {
item_json["Name"] = collection.rel_path;
}
if (!(collection.rel_path === "./")) { if (!(collection.rel_path === "./")) {
item_json["ID"] = collection.rel_path; item_json["ID"] = "./" + collection.rel_path;
} else if (!item_json["ID"]) { } else if (!item_json["ID"]) {
item_json["ID"] = collection.rel_path; item_json["ID"] = "./" + collection.rel_path;
} }
collection.collection_metadata.load_json(item_json, collection); collection.collection_metadata.load_json(item_json, collection);
} }
...@@ -80,7 +83,6 @@ module.exports = function() { ...@@ -80,7 +83,6 @@ module.exports = function() {
metadata = XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name]); metadata = XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name]);
for (var i = 0; i < metadata.length; i++) { for (var i = 0; i < metadata.length; i++) {
item_json = metadata[i]; item_json = metadata[i];
//console.log("JSON", item_json); //console.log("JSON", item_json);
item = new Item(); item = new Item();
item.load_json(item_json, collection); item.load_json(item_json, collection);
...@@ -109,7 +111,32 @@ module.exports = function() { ...@@ -109,7 +111,32 @@ module.exports = function() {
item_by_id: this.item_by_id, item_by_id: this.item_by_id,
item_by_type: this.item_by_type, item_by_type: this.item_by_type,
same_as: this.same_as, same_as: this.same_as,
make_id: function(id){
id = String(id);
if (id.match(/^https?:\/\//i)){
return id;
}
else {
return id;
}
},
add_id: function(item) {
var id = this.make_id(item.id);
this.id_lookup[id] = item;
},
lookup_id: function(id){
var id = this.make_id(id);
if (this.id_lookup[id]) {
return this.make_id(this.id_lookup[id].id);
}
},
lookup_name: function(name){
if (this.name_lookup[name]){
//console.log(name, this.make_id(this.name_lookup[name].id))
return this.make_id(this.name_lookup[name].id)
}
},
get_unique_catalog_name: function get_unique_catalog_name( get_unique_catalog_name: function get_unique_catalog_name(
dir, dir,
existing_catalogs = [] existing_catalogs = []
...@@ -131,7 +158,6 @@ module.exports = function() { ...@@ -131,7 +158,6 @@ module.exports = function() {
}, },
index_graph: function index_graph() { index_graph: function index_graph() {
// TODO - REMOVE this - we now have a helper for this stuff
this.item_by_id = {}; this.item_by_id = {};
this.item_by_path = {}; this.item_by_path = {};
this.item_by_type = {}; this.item_by_type = {};
...@@ -155,14 +181,10 @@ module.exports = function() { ...@@ -155,14 +181,10 @@ module.exports = function() {
? this.item_by_path["./"] ? this.item_by_path["./"]
: this.item_by_path["data/"]; : this.item_by_path["data/"];
}, },
to_json: function to_json(graph) { to_json: function to_json(graph) {
if (!this.collection_metadata) { if (!this.collection_metadata) {
this.collection_metadata = new Item(); this.collection_metadata = new Item();
} }
var collection_json = this.collection_metadata.to_json_ld_fragment(); var collection_json = this.collection_metadata.to_json_ld_fragment();
// Need to work out how to do this // Need to work out how to do this
/* collection_json["distribution"] = { /* collection_json["distribution"] = {
...@@ -184,7 +206,7 @@ module.exports = function() { ...@@ -184,7 +206,7 @@ module.exports = function() {
collection_json["hasPart"] = []; collection_json["hasPart"] = [];
} }
collection_json["hasPart"].push({ collection_json["hasPart"].push({
"@id": item.id "@id": this.make_id(item.id)
}); });
if (!item_json["name"]) { if (!item_json["name"]) {
item_json["name"] = item.id; item_json["name"] = item.id;
...@@ -198,16 +220,16 @@ module.exports = function() { ...@@ -198,16 +220,16 @@ module.exports = function() {
} }
} }
//Sub collections //Sub collections
this.children.forEach(function(child) { for (var child of this.children) {
child.to_json(graph); child.to_json(graph);
if (!collection_json["hasPart"]) { if (!collection_json["hasPart"]) {
collection_json["hasPart"] = []; collection_json["hasPart"] = [];
} }
collection_json["hasPart"].push({ collection_json["hasPart"].push({
"@id": child.collection_metadata.id "@id": this.make_id(child.collection_metadata.id)
//"@type": "@id" //"@type": "@id"
}); });
}); }
}, },
to_json_ld: function to_json_ld() { to_json_ld: function to_json_ld() {
...@@ -216,9 +238,9 @@ module.exports = function() { ...@@ -216,9 +238,9 @@ module.exports = function() {
"@graph": [], "@graph": [],
"@context": context "@context": context
}; };
this.to_json(json["@graph"]); this.to_json(json["@graph"]);
fs.writeFileSync("TEST.json", String(json));
json = JSON.parse(JSON.stringify(json)); json = JSON.parse(JSON.stringify(json));
for (var same of this.same_as) { for (var same of this.same_as) {
......
...@@ -23,7 +23,7 @@ const path = require("path"); ...@@ -23,7 +23,7 @@ const path = require("path");
const shell = require("shelljs"); const shell = require("shelljs");
const jsonld_helper = require("./jsonldhelper") const jsonld_helper = require("./jsonldhelper")
const filesize = require("filesize"); const filesize = require("filesize");
const sha1 = require('sha1');
const display_keys = [ const display_keys = [
"name", "name",
...@@ -189,7 +189,7 @@ module.exports = function () { ...@@ -189,7 +189,7 @@ module.exports = function () {
var up_link; var up_link;
var catalog_json_link = ""; var catalog_json_link = "";
var zip_link; var zip_link;
var name = this.root_node["name"]; var name = this.helper.get_name(this.root_node);
var json = ""; var json = "";
if (this.first_page) { if (this.first_page) {
...@@ -262,14 +262,14 @@ module.exports = function () { ...@@ -262,14 +262,14 @@ module.exports = function () {
*/ */
var td_ele = ""; var td_ele = "";
if (!part) { if (!part) {
// TODO: Not sure if this ever happens.. // TODO: Not sure if this ever happens..
} else if (k == "@type") { } else if (k == "@type") {
td_ele += this.format_header(part); td_ele += this.format_header(part);
} else if (k === "name") { } else if (k === "name") {
td_ele += ele("b"); td_ele += ele("b");
td_ele += part; td_ele += this.helper.get_name(item);
td_ele += close("b"); td_ele += close("b");
//td_ele.ele("a", part).att('href', item["@id"]).att('class', 'fa fa-external-link').att('title',item["@id"]); //td_ele.ele("a", part).att('href', item["@id"]).att('class', 'fa fa-external-link').att('title',item["@id"]);
} else if ( } else if (
...@@ -303,11 +303,11 @@ module.exports = function () { ...@@ -303,11 +303,11 @@ module.exports = function () {
td_ele += part; td_ele += part;
td_ele += close("a"); td_ele += close("a");
} else if (k == "@id") { } else if (k == "@id") {
if (item["@id"].match(/^https?:\/\//i)) { if (item["@id"] && item["@id"].match(/^https?:\/\//i)) {
td_ele += ele("a", { td_ele += ele("a", {
href: item["@id"], href: item["@id"],
class: "fa fa-external-link", class: "fa fa-external-link",
title: item.name title: this.helper.get_name(item)
}); });
td_ele += item["@id"]; td_ele += item["@id"];
...@@ -323,20 +323,26 @@ module.exports = function () { ...@@ -323,20 +323,26 @@ module.exports = function () {
"name" in this.helper.item_by_id[part["@id"]] "name" in this.helper.item_by_id[part["@id"]]
) { ) {
td_ele += this.helper.item_by_id[part["@id"]].name + " : " + this.helper.item_by_id[part["@id"]].value td_ele += this.helper.get_name(this.helper.item_by_id[part["@id"]]) + " : " + this.helper.item_by_id[part["@id"]].value
} }
else if (part["@id"] && this.helper.item_by_id[part["@id"]]) { else if (part["@id"] && this.helper.item_by_id[part["@id"]]) {
var target_name = this.helper.item_by_id[part["@id"]].name var target_name = this.helper.get_name(this.helper.item_by_id[part["@id"]])
? this.helper.item_by_id[part["@id"]].name ? this.helper.get_name(this.helper.item_by_id[part["@id"]])
: part["@id"]; : part["@id"];
var href = this.get_href(part["@id"], item["@id"]); var href = this.get_href(part["@id"], item["@id"]);
td_ele += ele("a", { href: href }); td_ele += ele("a", { href: href });
td_ele += target_name; td_ele += target_name;
td_ele += close("a"); td_ele += close("a");
} else { } else if (part["@id"] && part["@label"]) {
td_ele += `<a href="${part["@id"]}" class="fa fa-external-link">${part["@label"]}<a/>`
}
else if (part["@value"]) {
td_ele += part["@value"];
}else {
td_ele += part; td_ele += part;
} }
return td_ele; return td_ele;
}, },
get_up_path: function get_up_path(path) { get_up_path: function get_up_path(path) {
...@@ -360,26 +366,12 @@ module.exports = function () { ...@@ -360,26 +366,12 @@ module.exports = function () {
return defaults.html_file_name; return defaults.html_file_name;
} }
} }
var p = ""; var p = "";
p += defaults.html_multi_file_dirs + "/"; p += defaults.html_multi_file_dirs + "/";
p += "pairtree_root/"; id = sha1(id)
id = id.replace(/\^/g, "^5e") //arcp://ni,sha-256;f4OxZX_x_FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk/src/luhn.c
.replace(/ /g, "^20") p += id.replace(/(........)/g, '$1/').replace(/([^\/])$/, "$1/")
.replace(/"/g, "^22")
.replace(/</g, "^3c")
.replace(/\\/g, "^5c")
.replace(/\*/g, "^2a")
.replace(/\=/g, "^3d")
.replace(/\+/g, "^2b")
.replace(/>/g, "^3e")
.replace(/\|/g, "^7c")
.replace(/,/g, "^2c")
.replace(/\?/g, "^3f")
.replace(/\//g, "=")
.replace(/\:/g, "+")
.replace(/\./g,",")
p += id.replace(/(..)/g, '$1/').replace(/([^\/])$/, "$1/")
p += "index.html" p += "index.html"
return p return p
}, },
...@@ -454,7 +446,6 @@ module.exports = function () { ...@@ -454,7 +446,6 @@ module.exports = function () {
dataset_to_html: function dataset_to_html(node) { dataset_to_html: function dataset_to_html(node) {
// Turns any item into an HTML table // Turns any item into an HTML table
var html = ""; var html = "";
var keys = new Set(Object.keys(node)); var keys = new Set(Object.keys(node));
if (node["identifier"]) { if (node["identifier"]) {
...@@ -463,14 +454,13 @@ module.exports = function () { ...@@ -463,14 +454,13 @@ module.exports = function () {
keys.delete("identifier") keys.delete("identifier")
} }
} }
keys.delete("@id"); keys.delete("@id");
keys.delete("filename"); keys.delete("filename");
keys.delete("@reverse"); keys.delete("@reverse");
if (keys.has("encodingFormat")) { if (keys.has("encodingFormat")) {
keys.delete("fileFormat"); keys.delete("fileFormat");
} }
if (!this.first_page && node["@label"] && !node["name"]) { if (!this.first_page && node["@label"] && !this.helper.get_name(node)) {
html += node["@label"]; html += node["@label"];
for (let key of Object.keys(node)) { for (let key of Object.keys(node)) {
node[key] = this.helper.value_as_array(node[key]) node[key] = this.helper.value_as_array(node[key])
...@@ -493,14 +483,17 @@ module.exports = function () { ...@@ -493,14 +483,17 @@ module.exports = function () {
html += ele("hr"); html += ele("hr");
//keys.delete("@type"); //keys.delete("@type");
//keys.delete("hasPart"); //keys.delete("hasPart");
html += ele("tr"); if (!node["@id"].startsWith("./")) {
html += ele("th", { style: "white-space: nowrap; width: 1%;" }); html += ele("tr");
html += "@id"; html += ele("th", { style: "white-space: nowrap; width: 1%;" });
html += close("th"); html += "@id";
html += ele("td"); html += close("th");
html += this.format_property(node, "@id", node, false); html += ele("td");
html += close("td"); html += this.format_property(node, "@id", node, false);
html += close("tr"); html += close("td");
html += close("tr");
}
key_set = this.sort_keys(keys); key_set = this.sort_keys(keys);
// Show back-links last // Show back-links last
if ("@reverse" in node) { if ("@reverse" in node) {
...@@ -525,7 +518,7 @@ module.exports = function () { ...@@ -525,7 +518,7 @@ module.exports = function () {
out_path = this.out_path; out_path = this.out_path;
} else { } else {
out_path = this.get_html_path(node["@id"]); out_path = this.get_html_path(node["@id"]);
cite = node["name"]; cite = this.helper.get_name(node);
} }
this.write_html(out_path, html, node); this.write_html(out_path, html, node);
...@@ -546,7 +539,7 @@ module.exports = function () { ...@@ -546,7 +539,7 @@ module.exports = function () {
var datasets = []; var datasets = [];
var readmes = []; var readmes = [];
var html = ""; var html = "";
for (part of ["hasPart", "hasMember"]) { for (part of ["hasPart", "hasMember", "hasFile"]) {
if (node[part]) { if (node[part]) {
if (!Array.isArray(node[part])) { if (!Array.isArray(node[part])) {
node[part] = [node[part]]; node[part] = [node[part]];
...@@ -649,7 +642,7 @@ module.exports = function () { ...@@ -649,7 +642,7 @@ module.exports = function () {
this.text_citation = text_citation; this.text_citation = text_citation;
if (!text_citation) { if (!text_citation) {
this.text_citation = root_node["name"]; this.text_citation = this.helper.get_name(root_node);
} }
//if (root_node) { //if (root_node) {
body_el += this.dataset_to_html(root_node, true); body_el += this.dataset_to_html(root_node, true);
...@@ -672,7 +665,6 @@ module.exports = function () { ...@@ -672,7 +665,6 @@ module.exports = function () {
} //console.log(type); } //console.log(type);
} }
body_el += close("div"); body_el += close("div");
if (!this.multiple_files_dir) { if (!this.multiple_files_dir) {
this.write_html(this.out_path, body_el, this.helper.json_ld); this.write_html(this.out_path, body_el, this.helper.json_ld);
} }
......
...@@ -39,11 +39,11 @@ module.exports = function() { ...@@ -39,11 +39,11 @@ module.exports = function() {
this.nested_items = {}; this.nested_items = {};
function links_to_id(string, collection) { function links_to_id(string, collection) {
var links_to = undefined; var links_to = collection.lookup_id(string);
if (collection.id_lookup[string]) { //console.log("Linksto", string, links_to)
links_to = collection.id_lookup[string].id; if (!links_to) {
} else if (collection.name_lookup[string]) { links_to = collection.lookup_name(string)
links_to = collection.name_lookup[string].id; //console.log("looked up", string, links_to);
} }
return links_to; return links_to;
} }
...@@ -78,7 +78,9 @@ module.exports = function() { ...@@ -78,7 +78,9 @@ module.exports = function() {
to_json_ld_fragment: function to_json_ld_fragment() { to_json_ld_fragment: function to_json_ld_fragment() {
//console.log("Keys at start of output", Object.keys(this.properties)); //console.log("Keys at start of output", Object.keys(this.properties));
var frag = { "@id": String(this.id) }; var id = this.collection.make_id(this.id);
var frag = { "@id": String(id) };
//console.log("Setting id", this.id, this.name, this.nested_items); //console.log("Setting id", this.id, this.name, this.nested_items);
//console.log(" NAMELOOKUPS", this.collection.name_lookup); //console.log(" NAMELOOKUPS", this.collection.name_lookup);
...@@ -97,11 +99,12 @@ module.exports = function() { ...@@ -97,11 +99,12 @@ module.exports = function() {
var link_id = f.links_to var link_id = f.links_to
? f.links_to ? f.links_to
: links_to_id(f.data[k], this.collection); : links_to_id(f.data[k], this.collection);
//console.log("link_id", link_id, "relational", f.is_relational) //console.log("link_id", link_id, "relational", f.is_relational)
if (f.is_file) { if (f.is_file) {
this.is_file = true; this.is_file = true;
//types.push("schema:MediaObject"); //types.push("schema:MediaObject");
} else if (f.is_relational && link_id) { } else if (f.is_relational && link_id && link_id != id) {
//console.log("Looking for relations", f.data[k], this.collection) //console.log("Looking for relations", f.data[k], this.collection)
//console.log("GOT A LINK", f.name, f.id); //console.log("GOT A LINK", f.name, f.id);
frag[f.name].push({ "@id": link_id }); //, "@type" : "@id" frag[f.name].push({ "@id": link_id }); //, "@type" : "@id"
...@@ -117,9 +120,7 @@ module.exports = function() { ...@@ -117,9 +120,7 @@ module.exports = function() {