Commit 80be7a9f authored by PTSEFTON's avatar PTSEFTON
Browse files

Now uses ro-crate library including for rendering

parent 8b76c1f5
......@@ -21,16 +21,20 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
const Collection = require("./lib/collection.js");
const Index = require("./lib/index_html.js");
const Bag = require("./lib/bag.js");
//const args = require("minimist")(process.argv.slice(2));
// const ROCrate = require("ro-crate").ROCrate;
const ROCrate = require("../../working/ro-crate-js/lib/rocrate");
const path = require("path");
const shell = require("shelljs");
const program = require("commander");
const defaults = require("./lib/defaults.js");
const htmlFileName = defaults.html_file_name;
const metadata_file_name = defaults.metadata_json_file_name;
const JSON_helper = require("./lib/jsonldhelper.js")
const fs = require("fs")
var dirs = undefined;
program
.version("0.1.0")
.description(
......@@ -48,7 +52,6 @@ program
)
.option("-d, --depth [depth]", "Maximum depth to recurse into directories")
.option("-r, --recurse", "Recurse into directories looking for ro-crate-metadata_.xslx files")
.option("-s, --save", "Save ro-crate-metadata after trimming it (and adding missing context if necessary).")
.option("-c, --cratescript [cratesript]", "URL of Crate-script directtory")
.option(
"-u, --url [distro]",
......@@ -74,10 +77,8 @@ if (depth) {
var output_dir = dir;
c.read(dir, "./", undefined, depth);
c.to_json_ld().then(function() {
var json_helper = new JSON_helper()
json_helper.init(c.json_ld)
var metadata_path = path.join(c.dir, defaults.metadata_json_file_name)
fs.writeFileSync(metadata_path, JSON.stringify(json_helper.json_ld, null, 2 ));
fs.writeFileSync(metadata_path, JSON.stringify(c.json_ld, null, 2 ));
generateHTML(metadata_path);
});
}
......@@ -105,46 +106,44 @@ else {
async function generateHTML(metadata_path) {
// Save an HTML file
console.log("Generating html from exsiting " + metadata_path + " file");
console.log("Generating html from: " + metadata_path);
if (!path.isAbsolute(metadata_path)) {
metadata_path = path.join(process.cwd(), metadata_path);
}
var dir = path.dirname(metadata_path)
shell.rm("-rf", path.join(dir, "ro-crate-metadata_files"));
var json_helper = new JSON_helper();
// Loading this from in-situ jsonld file
json_helper.init(JSON.parse(fs.readFileSync(metadata_path)));
// Are we updating this?
if (program.save) {
fs.writeFileSync(metadata_path, JSON.stringify(json_helper.json_ld, null, 2 ));
}
var crate = new ROCrate(JSON.parse(fs.readFileSync(metadata_path)));
var root = crate.getRootDataset();
if (program.url){
// Add dowload info
if (!json_helper.root_node["distribution"]) {
json_helper.root_node["distribution"] = []
if (!root["distribution"]) {
root["distribution"] = []
}
json_helper.root_node["distribution"].push(
root["distribution"].push(
{
"@id": program.url
}
);
json_helper.json_ld["@graph"].push(
var graph = crate.getGraph();
graph.push(
{
"@id": program.url,
"contentUrl": program.url,
"@type": "DataDownload",
"encodingFormat": "zip"
});
json_helper.init(json_helper.json_ld);
crate = new roCrate(crate.getJson());
}
if (program.bag) {
// Bag the HTML
// TODO - GET THIS WORKING
var bagger = new Bag();
var dest = path.join(program.bag, path.basename(dir));
shell.rm("-rf", dest);
dir = bagger.bag(dir, program.bag, json_helper);
dir = bagger.bag(dir, program.bag, crate);
bagger.generate_bag_info();
bagger.save_bag_info();
if (!path.isAbsolute(dir)) {
......@@ -153,16 +152,12 @@ async function generateHTML(metadata_path) {
metadata_path = path.join(dir, metadata_file_name);
bagger.update();
// Reload new JSON
json_helper.json_ld = bagger.helper.json_ld;
crate = new ROCrate(crate.getJson());
}
console.log("PATH", metadata_path);
json_helper.init(JSON.parse(fs.readFileSync(metadata_path)));
var index_maker = new Index();
fs.writeFileSync("test.jsonld", JSON.stringify(json_helper.json_ld, null, 3));
index_maker.init(
json_helper.json_ld,
crate,
path.join(__dirname, "defaults/metadata_template.html")
);
......
var jsonldHelper = require("./jsonldhelper");
const defaults = require("./defaults")
class Checker {
constructor(json) {
this.helper = new jsonldHelper();
this.helper.init(json);
this.checklist = [];
}
hasContext() {
var checkItem = new CheckItem(
{
name: "Has @context",
message: "The json-ld has an appropriate context"
}
)
if (this.helper.json_ld["@context"] && this.helper.json_ld["@context"] === defaults.context) {
checkItem.status = true;
}
return checkItem;
}
hasRootDataset() {
var checkItem = new CheckItem(
{
name: "Has root Dataset",
message: "There is a JSON-LD item with @type of Dataset (http://schema.org/dataset)"
}
)
if (this.helper.root_node) {
checkItem.status = true;
}
return checkItem;
}
hasName() {
var checkItem = new CheckItem(
{
name: "Has name",
message: "The root Dataset has a name (http://schema.org/name)"
}
)
if (this.helper.root_node && this.helper.root_node.name && this.helper.root_node.name.length > 0) {
checkItem.status = true;
}
return checkItem;
}
hasCreator() {
var checkItem = new CheckItem(
{
name: "Has valid Creators",
message: "The root Dataset has at least one Creator (http://schema.org/creator) referred to by @id, and all creators have @type Person (http://schema.org/Person) or Organization (http://schema.org/Organization)"
}
)
if (this.helper.root_node && this.helper.root_node.creator) {
const targets = this.helper.value_as_array(this.helper.root_node.creator);
for (let t of targets) {
if (t["@id"] && this.helper.item_by_id[t["@id"]]) {
var creator = this.helper.item_by_id[t["@id"]];
const types = this.helper.value_as_array(creator["@type"]);
if (types.includes("Person") || types.includes("Organization")){
checkItem.status = true;
}
else {
checkItem.status = false;
break;
}
}
else {
checkItem.status = false;
break;
}
}
//checkItem.status = allTargetsOk;
}
return checkItem;
}
hasLicense() {
var checkItem = new CheckItem(
{
name: "Has a license ",
message: "The root Dataset has a License of type CreativeWork with a description"
}
)
if (this.helper.root_node && this.helper.root_node.license) {
const targets = this.helper.value_as_array(this.helper.root_node.license);
for (let t of targets) {
if (t["@id"]) {
var license = this.helper.item_by_id[t["@id"]];
const types = this.helper.value_as_array(license["@type"]);
// TODO check that license path is a path or URL is a URL and check name and description are legit
if (types.includes("CreativeWork") && license.name && license.description){
checkItem.status = true;
break;
}
}
}
}
return checkItem;
}
hasDatePublished() {
var checkItem = new CheckItem(
{
name: "Has a datePublished ",
message: "The root Dataset has a datePublished with ONE value which is an ISO 8601 format precision of at least a day"
}
)
var date = this.helper.value_as_array(this.helper.root_node.datePublished);
if (date.length != 1) {
checkItem.diagnostics = `Number of datePublished values is ${date.length} NOT 1`;
return checkItem;
}
checkItem.status =date[0].match(/^\d{4}-([0]\d|1[0-2])-([0-2]\d|3[01])/)
return checkItem;
}
hasContactPoint() {
var checkItem = new CheckItem(
{
name: "Has a contactPoint",
message: "The root Dataset has at least one contactPoint property which references a ContactPoint of type Customer Service"
}
)
if (this.helper.root_node && this.helper.root_node.contactPoint) {
const targets = this.helper.value_as_array(this.helper.root_node.contactPoint);
for (let t of targets) {
if (t["@id"]) {
var contact= this.helper.item_by_id[t["@id"]];
const types = this.helper.value_as_array(contact["@type"]);
if (types.includes("ContactPoint") && this.helper.value_as_array(contact.contactType).includes("customer service") && contact.email) {
checkItem.status = true;
break;
}
}
}
}
return checkItem;
}
check() {
var context = this.hasContext();
this.checklist.push(context);
var dataset = this.hasRootDataset();
this.checklist.push(dataset);
var name = this.hasName()
this.checklist.push(name);
var creator = this.hasCreator();
this.checklist.push(creator);
var license = this.hasLicense();
this.checklist.push(license);
var date = this.hasDatePublished();
this.checklist.push(date);
var contact = this.hasContactPoint();
this.checklist.push(contact);
this.isROCrate = context.status && dataset.status && name.status && creator.status && license.status && date.status;
// TODO:
// this.isDistributable
// this.isCitable
}
summarize() {
if (this.isROCrate) {
return "This is a valid RO-Crate";
}
else {
return "This is not a valid RO-Crate";
}
}
report() {
var report = [];
for (var item of this.checklist) {
const tick = item.status ? "✔️" : "";
report.push(`${tick} ${item.name}: ${item.message}`);
}
return report.join("\n");
}
}
class CheckItem {
constructor(data) {
this.name = data.name;
this.message = data.message;
this.status = false;
}
}
module.exports = Checker;
......@@ -20,8 +20,10 @@ var jsonld = require('jsonld');
var fs = require('fs');
var program = require('commander');
var defaults = require('./defaults');
const rocrate = require('ro-crate');
var XLSX = require('xlsx');
var path = require('path');
var RoCrate = rocrate.ROCrate;
const Property = require("./property.js");
......@@ -35,7 +37,6 @@ var fs = require('fs');
const Datacite = require('./datacite.js')
// Copy of default context
var context = JSON.parse(JSON.stringify(defaults.context));
module.exports = function() {
this.collection_metadata = new Item();
......@@ -65,15 +66,12 @@ module.exports = function() {
}
}
item_json["TYPE:"] = "Dataset";
item_json["path"] = collection.rel_path;
//item_json["path"] = collection.rel_path;
if(!item_json["Name"]) {
item_json["Name"] = collection.rel_path;
}
if (!(collection.rel_path === "./")) {
item_json["ID"] = "./" + collection.rel_path;
} else if (!item_json["ID"]) {
item_json["ID"] = "./" + collection.rel_path;
}
item_json["ID"] = collection.rel_path;
collection.collection_metadata.load_json(item_json, collection);
}
......@@ -155,11 +153,12 @@ module.exports = function() {
return potential_metadata_filename;
},
// TODO - get rid of this one - we have this function in RO-Crate
index_graph: function index_graph() {
this.item_by_id = {};
this.item_by_path = {};
this.item_by_type = {};
this.graph = this.json_ld["@graph"];
//this.graph = this.json_ld["@graph"];
for (let i = 0; i < this.graph.length; i++) {
var item = this.graph[i];
if (item["@id"]) {
......@@ -175,35 +174,27 @@ module.exports = function() {
this.item_by_type[item["@type"]].push(item);
}
}
this.root_node = this.item_by_path["./"]
? this.item_by_path["./"]
: this.item_by_path["data/"];
this.root_node = this.item_by_id["./"];
},
to_json: function to_json(graph) {
if (!this.collection_metadata) {
this.collection_metadata = new Item();
}
var collection_json = this.collection_metadata.to_json_ld_fragment();
// Need to work out how to do this
/* collection_json["distribution"] = {
"contentUrl": this.rel_path ,
"@type": "DataDownload",
"name": "Downloadable data distribution for : " + this.name
} */
graph.push(collection_json);
//console.log("COLLECTION METADATA", json);
for (var [key, item] of Object.entries(this.items)) {
item_json = item.to_json_ld_fragment();
// Keep track of whether to add this to the graph
var exists = true;
if (item.is_file) {
if (shell.test("-e", path.join(this.root_dir, item.id))) {
if (!collection_json["hasPart"]) {
collection_json["hasPart"] = [];
} else if (!Array.isArray(collection_json["hasPart"])) {
collection_json["hasPart"] = [collection_json["hasPart"]]
collection_json["hasPart"] = [collection_json["hasPart"]];
}
collection_json["hasPart"].push({
"@id": this.make_id(item.id)
......@@ -236,19 +227,16 @@ module.exports = function() {
to_json_ld: function to_json_ld() {
// Turn the entire collection into a JSON-LD document
json = {
"@graph": [],
"@context": context
};
this.to_json(json["@graph"]);
fs.writeFileSync("TEST.json", String(json));
json = JSON.parse(JSON.stringify(json));
this.crate = new RoCrate();
this.crate.index();
this.graph = this.crate.getGraph();
this.to_json(this.graph);
json = JSON.parse(JSON.stringify(this.crate.json_ld));
for (var same of this.same_as) {
json["@graph"].push(same)
}
//console.log(JSON.stringify(json, null, 2));
var collection = this;
promise = flattenit(json, this);
return promise.then(
......@@ -266,10 +254,7 @@ module.exports = function() {
fs.writeFileSync(
path.join(collection.dir, defaults.metadata_json_file_name),
JSON.stringify(
{
"@graph": flattenated["@graph"],
"@context": flattenated["@context"]
},
flattenated,
null,
2
),
......@@ -290,7 +275,6 @@ module.exports = function() {
);
},
read: function read(dir, rel_path = "./", parent = false, max_depth) {
//console.log("existing", parent.existing_metadatas)
if (max_depth) {
this.max_depth = max_depth;
} else {
......@@ -304,7 +288,6 @@ module.exports = function() {
this.id_lookup = parent.id_lookup;
this.existing_metadatas = parent.existing_metadatas;
this.root_dir = parent.root_dir;
//console.log(this.existing_metadatas);
} else {
this.depth = 1;
this.name_lookup = {};
......@@ -312,7 +295,6 @@ module.exports = function() {
this.existing_metadatas = [];
this.root_dir = dir;
}
//console.log("XXXXXX Collecting", dir, "Depth", this.depth);
this.children = [];
this.dir = dir;
this.rel_path = rel_path;
......
const Preview = require("./ro-crate-preview.js");
const Checker = require("./checker.js");
var meta;
var preview;
const check = function check() {
var checker = new Checker(meta);
checker.check();
$("div.check").html(`<details><summary>${checker.summarize()}</summary><a href='#___check____'><pre>${checker.report()}</pre></a></details>`);
}
async function load() {
meta = JSON.parse($("script[type='application/ld+json']").text());
preview = await new Preview(meta);
$("div.check").html("<button><a href='#___check____'>Check this crate</a></button>")
var hash = location.hash;
if (hash.startsWith('#___check')){
check()
} else if (hash) {
preview.display(unescape(hash.replace("#", "")));
}
else {
preview.display(preview.root["@id"]);
}
}
window.onhashchange = function() {
load()
}
$(document).ready(load);
......@@ -16,38 +16,37 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
const defaults = require("./defaults");
const Preview = require("./ro-crate-preview-wrapper")
const Preview = require("ro-crate").Preview
var ejs = require("ejs");
const fs = require("fs-extra");
const jsonld_helper = require("./jsonldhelper");
module.exports = function () {
return {
init: function init(crate_data, template_path) {
init: function init(crate, template_path) {
if (template_path) {
var temp = fs.readFileSync(template_path, { encoding: "utf8" });
this.template = ejs.compile(temp);
}
this.helper = new jsonld_helper();
this.helper.init(crate_data);
this.crate = crate;
this.crate.index();
//this.helper.add_back_links();
// A container for our page
},
make_index_html: async function make_index_html(zip_path, render_script) {
var body_el = "";
var name = this.helper.root_node.name;
var rootNode = this.crate.getRootDataset();
var name = rootNode.name;
if (!render_script) {
render_script = defaults.render_script;
}
if (zip_path) {
zip_path = `<a href='${zip_path}'>Download this Dataset</a>`;
}
const preview = new Preview(this.helper.json_ld );
const preview = new Preview(this.crate);
const summary = await preview.summarizeDataset();
var date = new Date();
var timestamp = date.getTime();
......@@ -61,7 +60,7 @@ module.exports = function () {
time_stamp: timestamp,
ROCrate_version: defaults.ROCrate_version,
spec_id: defaults.DataCrate_Specification_Identifier,
json_ld: JSON.stringify(this.helper.json_ld, null, 2),
json_ld: JSON.stringify(this.crate.getJson(), null, 2),
render_script: render_script
})
}
......
......@@ -127,8 +127,6 @@ module.exports = function() {
helper.push_value(frag, "identifier", id.replace(/https?:\/\//i, ""))
}
}
this.json_ld_fragment = frag;
//console.log(frag);
return frag;
......@@ -152,9 +150,9 @@ module.exports = function() {
//console.log("FILE ", this.collection.path, value);
this.id = path.join(this.collection.rel_path, value);
this.types.push("File");
var pr = new metadata_property_name();
pr.parse("path", this.id);
this.properties[pr.name] = pr;
//var pr = new metadata_property_name();
//pr.parse("path", this.id);
//this.properties[pr.name] = pr;
this.is_file = value;
} else if (property.is_id) {
//console.log("Got an ID", value);
......
/*
This is part of Calcyte a tool for implementing the DataCrate data packaging
spec. Copyright (C) 2018-2019 University of Technology Sydney
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.