Commit 8b76c1f5 authored by PTSEFTON's avatar PTSEFTON
Browse files

Changed max number of files to list to 30

parent e619ab38
......@@ -4,19 +4,19 @@ This is a work-in-progress port of the python-based [Calcyte tool](https://codei
## Status
This is Beta code.
This is Alpha code.
## About
Calcyte is a toolkit to implement the [DataCrate] specification:
Calcyte is a toolkit to implement the [RO-Crate] specification:
1. Generating HTML from DataCrate CATALOG.json files.
1. Generating HTML from RO-Crate `ro-crate-metadata.jsonld` files.
2. Managing metadata for collections of content via automatically generated
spreadsheets, to create CATALOG.json files
spreadsheets (CATALOG.xlsx), to create `ro-crate-metadata.jsonld` files
3. Packaging data in BagIt format, and optionally zipping it.
Calcyte targets the [Draft DataCrate Packaging format v0.3](https://github.com/UTS-eResearch/datacrate/blob/master/spec/0.3/data_crate_specification_v0.3.md).
This version of Calcyte targets the [R0-Crate Spec, v0.2](https://github.com/UTS-eResearch/RO-Crate/blob/master/spec/0.3/data_crate_specification_v0.3.md).
## Installation
......@@ -28,7 +28,9 @@ Calcyte targets the [Draft DataCrate Packaging format v0.3](https://github.com/U
- Install Siegfreid using the [instructions](https://github.com/richardlehane/siegfried/wiki/Getting-started).
- Get the code:
git clone https://code.research.uts.edu.au/eresearch/CalcyteJS.git
git clone --single-branch --branch <branchname> https://code.research.uts.edu.au/eresearch/CalcyteJS.git
- Link the binary for development use:
......@@ -36,32 +38,41 @@ Calcyte targets the [Draft DataCrate Packaging format v0.3](https://github.com/U
## Usage / instructions
Calcyfy creates RO-Crates whith and HTML entry-point. This relies on
Get a web server running on local host:
```
```
Usage:
```
> ./calcyfy
Usage: calcyfy [options] <directories...>
Usage: calcyfy [options] <directories...>
Generates DataCrate HTML for CATALOG.JSON files. Pass a list of directories. To create Spreadsheet files for entring metadata use -d or -r.
Generates DataCrate HTML for ro-crate-metadata.jsonld files. Pass a list of directories. To create Spreadsheet files for entring metadata use -d or -r.
Options:
-V, --version output the version number
-b, --bag [bag-dir] Create Bagit Bag(s) under [bag-dir])
-n, --no No Citation - only applies ith --bag
-z, --zip Create a zipped version of the bag - only applies with --bag
-d, --depth Maximum depth to recurse into directories
-r, --recurse Recurse into directories - up to 10
-u, --url [distro] Distribution URL
-h, --help output usage information
-V, --version output the version number
-b, --bag [bag-dir] Create Bagit Bag(s) under [bag-dir])
-n, --no No Citation - only applies ith --bag
-z, --zip Create a zipped version of the bag - only applies with --bag
-d, --depth [depth] Maximum depth to recurse into directories
-r, --recurse Recurse into directories looking for ro-crate-metadata_.xslx files
-s, --save Save ro-crate-metadata after trimming it (and adding missing context if necessary).
-h, --htmlscript [htmlscript] HTML render script
-u, --url [distro] Distribution URL
-h, --help output usage information
```
To run Calcyte on a group of directories pass it a list of directories
To generate an HTML file for a CATALOG.json file in ./dir
To generate an HTML file for an `ro-crate-metadata.jsonld` file in ./dir
```
calcyfy dir
......@@ -70,13 +81,15 @@ calcyfy dir
One directory:
```
calcyfy test_data/Glop_Pot -r
calcyfy -r test_data/sample/
calcyfy test_data/Glop_Pot -r -h http://localhost:8080/compiled
```
This will:
- Traverse the entire Glop_Pot directory, and generate or update CATALOG_name.xlsx files.
- Create or update the `test_data/Glop_Pot/CATALOG.json` file
- Create a *DataCrate Website* with entry-point `test_data/Glop_Pot/CATALOG.html`
- Create or update the `test_data/Glop_Pot/`ro-crate-metadata.jsonld`` file
- Create a *RO-Crate Website* with entry-point `test_data/Glop_Pot/CATALOG.html`
All the sample directories:
......@@ -89,12 +102,12 @@ Calcyte will generate:
- a CATALOG\_$dir.xlsx file in each directory (this is for humans to fill in with
metadata about the data)
- An index.html file summarizing the data using metadata from CATALOG\_$dir.xlsx
- An `ro-crate-preview.html` file summarizing the data using metadata from CATALOG\_$dir.xlsx
- A CATALOG.json file containing JSON-LD metadata derived from the CATALOG\* files plus some basic file-format information.
- An `ro-crate-metadata.jsonld` file containing JSON-LD metadata derived from the CATALOG\* files plus some basic file-format information.
See the examples in `test_data`.
TODO: Instructions for filling in the CATALOG files.
[datacrate]: https://github.com/UTS-eResearch/datacrate
[RO-Crate]: https://github.com/UTS-eResearch/RO-Crate
......@@ -34,7 +34,7 @@ var dirs = undefined;
program
.version("0.1.0")
.description(
"Generates DataCrate HTML for ro-crate-metadata.jsonld files. Pass a list of directories. To create Spreadsheet files for entring metadata use -d or -r."
"Generates RO-Crate HTML for ro-crate-metadata.jsonld files. Pass a list of directories. To create Spreadsheet files for entering metadata use -d or -r."
)
.arguments("<directories...>")
.action(function(directories) {
......@@ -49,8 +49,7 @@ program
.option("-d, --depth [depth]", "Maximum depth to recurse into directories")
.option("-r, --recurse", "Recurse into directories looking for ro-crate-metadata_.xslx files")
.option("-s, --save", "Save ro-crate-metadata after trimming it (and adding missing context if necessary).")
.option("-h, --htmlscript [htmlscript]", "HTML render script")
.option("-c, --cratescript [cratesript]", "URL of Crate-script directtory")
.option(
"-u, --url [distro]",
"Distribution URL"
......@@ -71,7 +70,6 @@ if (depth) {
dirs.forEach(function(dir) {
if (shell.test("-d", dir)) {
shell.rm("-rf", path.join(dir, "ro-crate-metadata_files"));
var c = new Collection();
var output_dir = dir;
c.read(dir, "./", undefined, depth);
......@@ -169,10 +167,10 @@ async function generateHTML(metadata_path) {
);
fs.writeFileSync(path.join(dir, htmlFileName),
await index_maker.make_index_html(program.url, program.htmlscript));
await index_maker.make_index_html(program.url, program.cratescript));
if (program.zip) {
if (program.zip && program.bag) {
shell.cd(bagger.target_dir);
var zipname = path.join("../", path.basename(bagger.target_dir) + ".zip");
shell.rm("-f", zipname);
......
......@@ -211,7 +211,7 @@ module.exports = Checker;
 
},{"./defaults":2,"./jsonldhelper":4}],2:[function(require,module,exports){
(function (__dirname){
/* This is part of Calcyte a tool for implementing the RO-Crate data packaging
/* This is part of Calcyte a tool for implementing the RO-Crate data packaging
spec. Copyright (C) 2018 University of Technology Sydney
 
This program is free software: you can redistribute it and/or modify
......
No preview for this file type
......@@ -289,7 +289,7 @@ module.exports = function() {
}
);
},
read: function read(dir, rel_path = "./", parent = false, max_depth = 1) {
read: function read(dir, rel_path = "./", parent = false, max_depth) {
//console.log("existing", parent.existing_metadatas)
if (max_depth) {
this.max_depth = max_depth;
......@@ -345,7 +345,7 @@ module.exports = function() {
//console.log("CATALOGS", metadatas)
//TODO - make this configurable
if (metadatas.length === 0) {
console.log("Making new metadata");
//console.log("Making new metadata");
var metadata_file = !parent
? `${defaults.metadata_root_name}.xlsx`
: `${defaults.metadata_root_name}_subdir.xlsx`;
......@@ -354,7 +354,7 @@ module.exports = function() {
this.existing_metadatas
);
this.existing_metadatas.push(new_metadata_file);
console.log("EXISTING AT THIS POINT", this.existing_metadatas);
//console.log("EXISTING AT THIS POINT", this.existing_metadatas);
metadatas = [new_metadata_file];
fs.writeFileSync(
path.join(dir, new_metadata_file),
......@@ -481,7 +481,7 @@ module.exports = function() {
for (var i = 0; i < subdirs.length; i++) {
if (this.depth < this.max_depth) {
var child = new module.exports();
//console.log("making new child:", subdirs[i]);
//console.log("making new child:", this.depth, this.max_depth, subdirs[i]);
child.read(
path.join(dir, subdirs[i]),
path.join(this.rel_path, subdirs[i]),
......
/* This is part of Calcyte a tool for implementing the RO-Crate data packaging
/* This is part of Calcyte a tool for implementing the RO-Crate data packaging
spec. Copyright (C) 2018 University of Technology Sydney
This program is free software: you can redistribute it and/or modify
......@@ -23,7 +23,7 @@ const datacite_file_name = "datacite.xml";
const metadata_json_file_name = `${ro_crate_name}.jsonld`;
const html_file_name = "ro-crate-preview.html";
const max_depth = "10"; // Number of dirs to recurse into
const max_files_in_dir = "100"; // Don't list files in a directory if there are more than this
const max_files_in_dir = "30"; // Don't list files in a directory if there are more than this
const ignore_file_regex = new RegExp(
`(^~)|(^\\.)|(${datacite_file_name})|^${html_file_name}$|^${metadata_json_file_name}$`
);
......
......@@ -47,7 +47,7 @@ module.exports = function () {
if (zip_path) {
zip_path = `<a href='${zip_path}'>Download this Dataset</a>`;
}
const preview = new Preview(this.helper.json_ld);
const preview = new Preview(this.helper.json_ld );
const summary = await preview.summarizeDataset();
var date = new Date();
var timestamp = date.getTime();
......
......@@ -94,7 +94,7 @@ describe("JSON-LD helper simple tests", function () {
assert.equal(helper.item_by_id["4"]["isPartOf"][0]["@id"], "1")
assert.equal(helper.item_by_id["2"]["@reverse"]["creator"][0]["@id"], "1")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment