Commit 19701ca7 authored by PTSEFTON's avatar PTSEFTON
Browse files

Moving to dynamically generated HTML

parent a8028b98
......@@ -20,7 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
const Collection = require("./lib/collection.js");
const Index = require("./lib/index_html.js");
const Datacite = require("./lib/datacite.js");
const Bag = require("./lib/bag.js");
//const args = require("minimist")(process.argv.slice(2));
const path = require("path");
......@@ -47,9 +46,10 @@ program
"-z, --zip",
"Create a zipped version of the bag - only applies with --bag"
)
.option("-d, --depth", "Maximum depth to recurse into directories")
.option("-d, --depth [depth]", "Maximum depth to recurse into directories")
.option("-r, --recurse", "Recurse into directories looking for ro-crate-metadata_.xslx files")
.option("-s, --save", "Save ro-crate-metadata after trimming it (and adding missing context if necessary).")
.option("-h, --htmlscript [htmlscript]", "HTML render script")
.option(
"-u, --url [distro]",
......@@ -63,7 +63,6 @@ if (program.recurse) {
if (program.depth) {
depth = program.depth;
}
var citer = new Datacite();
if (!program.rawArgs.length || !dirs) program.help();
// TODO: Refactor this by if statement - there is a lot of repetition
......@@ -106,8 +105,8 @@ else {
})
}
function generateHTML(metadata_path) {
async function generateHTML(metadata_path) {
// Save an HTML file
console.log("Generating html from exsiting " + metadata_path + " file");
if (!path.isAbsolute(metadata_path)) {
metadata_path = path.join(process.cwd(), metadata_path);
......@@ -116,11 +115,14 @@ function generateHTML(metadata_path) {
shell.rm("-rf", path.join(dir, "ro-crate-metadata_files"));
var json_helper = new JSON_helper();
// Loading this from in-situ jsonld file
json_helper.init(JSON.parse(fs.readFileSync(metadata_path)));
// Are we updating this?
if (program.save) {
fs.writeFileSync(metadata_path, JSON.stringify(json_helper.json_ld, null, 2 ));
}
if (program.url){
// Add dowload info
if (!json_helper.root_node["distribution"]) {
json_helper.root_node["distribution"] = []
}
......@@ -136,56 +138,47 @@ function generateHTML(metadata_path) {
"@type": "DataDownload",
"encodingFormat": "zip"
});
// TODO - GET RID OF THIS ABOMINATION!
json_helper.json_ld["@context"]["DataDownload"] = "https://schema.org/DataDownload"
json_helper.json_ld["@context"]["distribution"] = "https://schema.org/distribution"
json_helper.init(json_helper.json_ld);
json_helper.init(json_helper.json_ld);
}
if (program.bag) {
//Bag the HTML
// Bag the HTML
var bagger = new Bag();
var dest = path.join(program.bag, path.basename(dir));
var zipname
shell.rm("-rf", dest);
dir = bagger.bag(dir, program.bag, json_helper);
bagger.generate_bag_info()
bagger.save_bag_info()
bagger.generate_bag_info();
bagger.save_bag_info();
if (!path.isAbsolute(dir)) {
dir = path.join(process.cwd(), dir);
}
metadata_path = path.join(dir, metadata_file_name);
if (!program.no) {
shell.mkdir("-p", path.join(dest, "metadata"));
var text_citation = citer.make_citation(
metadata_path,
path.join(dir, "metadata", "datacite.xml")
);
}
bagger.update();
// Reload new JSON
json_helper.json_ld = bagger.helper.json_ld;
}
console.log("PATH", metadata_path);
json_helper.init(JSON.parse(fs.readFileSync(metadata_path)));
var index_maker = new Index();
fs.writeFileSync("test.jsonld", JSON.stringify(json_helper.json_ld, null, 3));
index_maker.init(
json_helper.json_ld,
path.join(dir, defaults.html_file_name),
path.join(__dirname, "defaults/metadata_template.html")
);
if (program.bag) {
bagger.update();
if (program.zip) {
shell.cd(bagger.target_dir);
zipname = path.join("../", path.basename(bagger.target_dir) + ".zip");
shell.rm("-f", zipname);
shell.cd(bagger.target_dir);
index_maker.make_index_html(text_citation, "");
fs.writeFileSync(path.join(dir, defaults.html_file_name),
await index_maker.make_index_html(program.url, program.htmlscript));
shell.exec("zip -r " + zipname + " " + "*");
}
if (program.zip) {
shell.cd(bagger.target_dir);
var zipname = path.join("../", path.basename(bagger.target_dir) + ".zip");
shell.rm("-f", zipname);
shell.cd(bagger.target_dir);
shell.exec("zip -r " + zipname + " " + "*");
}
index_maker.make_index_html(text_citation, zipname);
}
\ No newline at end of file
......@@ -2,41 +2,23 @@
<html>
<head>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
<style>
table {
width: 90%;
text-align: left;
vertical-align: top;
margin-bottom: 2em;
}
td {
margin-right: 2em;
white-space: normal;
<script type="application/ld+json">
<%- json_ld %>
</script>
}
details {
margin-left: 2em;
}
.break {
word-break: break-all;
}
div.collection {
margin-left: +1em;
}
</style>
<script type="application/ld+json">
<%- json_ld %>
</script>
<script>
meta = <%- json_ld %>
</script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
<script src="<%= render_script %>/crate.js"></script>
<link rel="stylesheet"
href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"
......@@ -46,30 +28,15 @@
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<meta charset='utf-8'/>
<style>
<script src="http://localhost:8080/lib/render.js"></script>
<script>
function load() {
var hash = location.hash;
if (hash) {
display(unescape(hash.replace("#", "")));
}
else {
display(root["@id"]);
}
table.table {
padding-bottom: 300px;
}
window.onhashchange = function() {
load()
}
$(document).ready(load);
</script>
</style>
</head>
<body>
<nav class="navbar navbar-inverse">
......@@ -84,26 +51,22 @@ window.onhashchange = function() {
<h4 class="citation"><%- citation %></h3>
<h3 class="name">NAME - TODO</h4>
<h4><%- zip_link %></h4>
</div>
<a href="./ro-crate-metadata.jsonld">Download all the metadata for this Dastaset in JSON-LD format</a>
</div>
<%- metadata_json_link %>
<div class="meta">
<table width="100%">
</table>
<div id="summary">
<%- html %>
</div>
<%- html %>
<p>This file was created at <%- time_stamp %> by
<a href='https://code.research.uts.edu.au/eresearch/calcytejs'>
Calcyte</a> which implements the <a href='<%- spec_id %>'>
Draft DataCrate Packaging format</a>, version <%- DataCrate_version %>
Draft RO-Crate format</a>, version <%- ROCrate_version %>
</p>
......
......@@ -15,7 +15,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* Bags a datacrate wich already has a CATALOG.json file */
/* Bags a datacrate wich already has a metadata.json file */
const path = require("path");
const shell = require("shelljs");
......@@ -26,7 +26,7 @@ const defaults = require("./defaults.js");
module.exports = function() {
return {
target_dir: this.target_dir,
catalog_path: this.catalog_path,
metadata_path: this.metadata_path,
generate_bag_info: function generate_bag_info() {
......@@ -80,17 +80,23 @@ module.exports = function() {
update: function update_bag_tags() {
shell.exec("bagit updatetagmanifests " + this.target_dir);
},
fix_paths: function fix_paths(catalog_path) {
if (!path.isAbsolute(catalog_path)) {
catalog_path = path.join("./", catalog_path);
// Move to /data
fix_paths: function fix_paths(metadata_path) {
if (!path.isAbsolute(metadata_path)) {
metadata_path = path.join("./", metadata_path);
}
var catalog = this.helper.json_ld;
for (let item of catalog["@graph"]) {
var metadata = this.helper.json_ld;
for (let item of metadata["@graph"]) {
if (item["path"]) {
var p = this.helper.value_as_array(item["path"])[0];
var new_p = path.join("data", p);
item["path"] = [new_p];
var new_p = item["@path"];
if (p != "./") {
new_p = path.join("data", p);
item["path"] = new_p;
}
if (item["@id"] === p) {
this.helper.update_id(item, new_p)
}
......@@ -99,11 +105,11 @@ module.exports = function() {
}
fs.writeFileSync(
catalog_path,
metadata_path,
JSON.stringify(this.helper.json_ld, null, 2),
function(err) {
if (err) {
return console.log(err, "Error writing in", catalog_path);
return console.log(err, "Error writing in", metadata_path);
}
}
);
......@@ -111,17 +117,18 @@ module.exports = function() {
bag: function bag(source_dir, bag_dir, helper) {
// TODO Generate a list of all files
// FOR NOW: delete CATALOG.json and index.html
// FOR NOW: delete metadata.json and index.html
// Generate bag info later
this.helper = helper;
var tmpobj = tmp.dirSync();
var bag_name = path.basename(source_dir);
var target_dir = path.join(bag_dir, bag_name);
shell.cp(
path.join(source_dir, defaults.catalog_json_file_name ),
path.join(tmpobj.name,"CATALOG_backup.json")
path.join(source_dir, defaults.metadata_json_file_name ),
path.join(tmpobj.name,"metadata_backup.json")
);
shell.cp(path.join(source_dir, defaults.catalog_json_file_name), tmpobj.name);
shell.cp(path.join(source_dir, defaults.metadata_json_file_name), tmpobj.name);
shell.rm(path.join(source_dir, defaults.html_file_name)); // Don't worry, welll make a new one
shell.exec(
"bagit create --excludebaginfo " +
......@@ -129,14 +136,11 @@ module.exports = function() {
" " +
path.join(source_dir, "*")
);
shell.rm(path.join(target_dir, "data", defaults.catalog_json_file_name), target_dir);
shell.rm(path.join(target_dir, "data", defaults.metadata_json_file_name), target_dir);
shell.exec("bagit update " + target_dir);
shell.cp(path.join(tmpobj.name, defaults.catalog_json_file_name), target_dir);
shell.cp(
path.join(__dirname, "..", defaults.DataCrate_profile_file),
target_dir
);
this.fix_paths(path.join(target_dir, defaults.catalog_json_file_name));
shell.cp(path.join(tmpobj.name, defaults.metadata_json_file_name), target_dir);
this.fix_paths(path.join(target_dir, defaults.metadata_json_file_name));
this.target_dir = target_dir;
return target_dir;
}
......
/* This is part of Calcyte a tool for implementing the DataCrate data packaging
/* This is part of Calcyte a tool for implementing the RO-Crate data packaging
spec. Copyright (C) 2018 University of Technology Sydney
This program is free software: you can redistribute it and/or modify
......@@ -29,11 +29,11 @@ const ignore_file_regex = new RegExp(
);
const ignore_dir_regex = new RegExp(`(^${html_multi_file_dirs}$)|(^\\.)`);
const BagIt_Profile_Identifier =
"https://raw.githubusercontent.com/UTS-eResearch/datacrate/master/spec/1.0/profile-datacrate-v1.0.json";
const DataCrate_Specification_Identifier =
"https://raw.githubusercontent.com/UTS-eResearch/RO-Crate/master/spec/1.0/profile-datacrate-v1.0.json";
const ROCrate_Specification_Identifier =
"https://github.com/UTS-eResearch/datacrate/blob/master/spec/1.0/data_crate_specification_v1.0.md";
const DataCrate_version = "1.0";
const DataCrate_profile_file = "defaults/profile-datacrate-v" + DataCrate_version + ".json"
const ROCrate_version = "0.2";
const ROCrate_profile_file = "defaults/profile-datacrate-v" + ROCrate_version + ".json"
const path = require("path")
const DEFAULTS = path.join(__dirname, '../defaults');
......@@ -41,6 +41,7 @@ const defaults_dir = DEFAULTS;
const context = "https://raw.githubusercontent.com/ResearchObject/ro-crate/master/docs/0.2-DRAFT/context.json";
const default_context = context;
const metadata_template = path.join(DEFAULTS, 'metadata_template.html');
const render_script = "https://code.research.uts.edu.au/eresearch/CalcyteJS/raw/feature/or-crate/lib/render.js";
const display_keys = [
"name",
......@@ -64,7 +65,7 @@ const display_keys = [
"@reverse",
];
// DataCrate specific terms which have inverses
// RO-Crate specific terms which have inverses
const back_links = {
hasFile: "fileOf",
hasPart: "isPartOf",
......@@ -88,13 +89,14 @@ module.exports = {
ignore_dir_regex: ignore_dir_regex,
max_files_in_dir: max_files_in_dir,
BagIt_Profile_Identifier: BagIt_Profile_Identifier,
DataCrate_Specification_Identifier: DataCrate_Specification_Identifier,
DataCrate_version: DataCrate_version,
ROCrate_Specification_Identifier: ROCrate_Specification_Identifier,
ROCrate_version: ROCrate_version,
back_links: back_links,
back_back_links: back_back_links,
DataCrate_profile_file: DataCrate_profile_file,
ROCrate_profile_file: ROCrate_profile_file,
context: context,
defaults_dir: defaults_dir,
metadata_template: metadata_template,
display_keys: display_keys
display_keys: display_keys,
render_script: render_script
};
......@@ -15,79 +15,51 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
const defaults = require("./defaults.js");
var fs = require("fs");
const defaults = require("./defaults");
const Preview = require("./ro-crate-preview")
var ejs = require("ejs");
const fs = require("fs-extra");
//context = require("../defaults/context.json");
const path = require("path");
const shell = require("shelljs");
const jsonld_helper = require("./jsonldhelper")
const filesize = require("filesize");
const sha1 = require('sha1');
const back_links = {
hasFile: "fileOf",
hasPart: "isPartOf",
hasMember: "memberOf"
};
const jsonld_helper = require("./jsonldhelper");
const dont_back_link = new Set(Object.values(back_links));
// TODO - Put this in a utility function
const arrayify = function arrayify(something, callback) {
if (!Array.isArray(something)) {
something = [something];
}
return callback(something);
};
const ele = function ele(name, atts = {}) {
t = "<" + name;
for (let key in atts) {
t += " " + key + " = '" + atts[key] + "'";
}
t += "\n>";
return t;
};
const close = function close(name) {
return "</" + name + "\n>";
};
module.exports = function () {
return {
init: function init(crate_data, out_path, template_path) {
console.log("TEMPLATE PATH", template_path)
init: function init(crate_data, template_path) {
if (template_path) {
var temp = fs.readFileSync(template_path, { encoding: "utf8" });
this.template = ejs.compile(temp);
}
this.out_path = out_path;
this.helper = new jsonld_helper();
this.helper.init(crate_data);
this.helper.add_back_links();
// A container for our page
},
make_index_html: function make_index_html(text_citation, zip_path) {
make_index_html: async function make_index_html(zip_path, render_script) {
var body_el = "";
this.zip_path = zip_path;
fs.writeFileSync(
this.out_path,
this.template({
html: "TODO",
citation: this.text_citation,
metadata_json_link: "metadata_json_link",
zip_link: zip_path,
up_link: "up_link",
time_stamp: "time",
DataCrate_version: defaults.DataCrate_version,
spec_id: defaults.DataCrate_Specification_Identifier,
json_ld: JSON.stringify(json, null, 2)
})
);
if (!render_script) {
render_script = defaults.render_script;
}
if (zip_path) {
zip_path = `<a href='${zip_path}'>Download this Dataset</a>`;
}
const preview = new Preview(json);
const summary = await preview.summarizeDataset()
return this.template({
html:summary.html(),
citation: this.text_citation,
zip_link: zip_path,
up_link: "",
time_stamp: "time",
ROCrate_version: defaults.ROCrate_version,
spec_id: defaults.DataCrate_Specification_Identifier,
json_ld: JSON.stringify(this.helper.json_ld, null, 2),
render_script: render_script
})
}
}
}
......@@ -15,64 +15,31 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* JSON-LD utility and lookup functions */
var jsonld = require("jsonld");
var defaults = require("./defaults.js")
var URI = require("uri-js")
const path = require("path")
var defaults = require("./defaults.js");
// TODO
// Context looker-upper
class jsonldHelper {
constructor () {
}
module.exports = function () {
return {
item_by_path: this.item_by_path,
item_by_id: this.item_by_id,
item_by_type: this.item_by_type,
get_uri_for_term: function (term) {
if (this.json_ld["@context"][term]) {
if (this.json_ld["@context"][term]["@id"]) {
term = this.json_ld["@context"][term]["@id"]
}
else if (!this.json_ld["@context"][term]["@type"]) {
term = this.json_ld["@context"][term]
}
}
var url = URI.parse(term)
// Looks like a URL
if (url.scheme) {
if (!url.host) {
term = this.get_uri_for_term(url.scheme) + url.path
}
return (term)
}
else {
return null
}
},
flatten: function(json) {
flatten(){
var promises = jsonld.promises;
console.log("************************* New context ************************", defaults.context);
var promise = promises.flatten(json, defaults.context); //require("../defaults/context.json")); //,
return promise.then(
(flattened) => {
this.init(flattened);
this.trim_context();
});
},
}
reference_to_item: function (node) {
reference_to_item(node) {
// Check if node is a reference to something else
// If it is, return the something else
if (node["@id"] && this.item_by_id[node["@id"]]) {
......@@ -81,9 +48,9 @@ module.exports = function () {
else {
return null
}
},
}
value_as_array: function (value) {
value_as_array(value) {
if (!value) {
return []
}
......@@ -92,9 +59,9 @@ module.exports = function () {
} else {
return value;
}
},
}
push_value: function(item, key, value) {
push_value(item, key, value) {
if (item[key]) {
if (!Array.isArray(item[key])) {
item[key] = [item[key]]
......@@ -105,8 +72,8 @@ module.exports = function () {
item[key] = [value]
}