diff --git a/README.md b/README.md index de6323beabd3c1e8fb97287572a3b8a01da664dd..575aabf5d46bc0cae1178e01c79152950f100f3b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,64 @@ # ocfl-demos +This is a repository for scripts and demos for the project [FAIR Simple Scalable Static Research Data Repository Demonstrator](https://www.eresearch.uts.edu.au/2019/06/07/ardc_ocfl.htm). + +Some of the data is UTS-only but the scripts are available for re-use. + +## Audience + +This is for NodeJS developers at UTS, and other interested parties capable of coding in node-js and advanced problem solving. + +## Conventions + +This guide assumes that you are working in `~/working`. + +## Installation + +To install calcyte-js from git (because you need the test data): + +- Follow [the instructions](https://code.research.uts.edu.au/eresearch/CalcyteJS) - you do not need bagit, so that step can be skipped. + +- Get this repository and install it: + + `git clone https://code.research.uts.edu.au/eresearch/ocfl-demos.git ~/working` + + `cd ~/working/ocfl-demos` + + `npm install .` + + +## Usage +To make an OCFL repository from the calcyte examples: + +- `make sample CALCYFY_PATH=~/working/calcyte.js/ REPO_PATH=~/working/ocfl-demo` + + That will: + - Generate an RO-Crate at `~/working/calcyte.js/test_data/sample` + - Create an OCFL repository at `~/working/ocfl-demo` if it does not already exist + - Deposit the RO-Crate into the repository + + NOTE: + - Subsequent runs will add new versions of the same crate to the repository. + +To make all the example from calcyte: +- `make all CALCYFY_PATH=~/working/calcyte.js/ REPO_PATH=~/working/ocfl-demo` + +This will: +- Generate RO-Crates for 4 examples at `~/working/calcyte.js/test_data/` +- Create an OCFL repository at `~/working/ocfl-demo` if it does not already exist +- Deposit the RO-Crate into the repository +- Generate an index file + + +## TODO + +## Add more examples such as Cameron Neylon's dataset + +### Port legacy datasets from the UTS Research Data Repository + +### Run with data downloaded from Research Data Australia + +### Add in sample data from Omeka (Farms to freeways and Dharmae) + + + diff --git a/make-ocfl-index.js b/make-ocfl-index.js new file mode 100644 index 0000000000000000000000000000000000000000..0fef97fddea91c0e2949e8b8ecbbaf8cb0f818a3 --- /dev/null +++ b/make-ocfl-index.js @@ -0,0 +1,71 @@ +// Simple Demo script to show usage + +const path = require('path'); +const fs = require('fs-extra'); +const ocfl = require("ocfl"); +console.log(ocfl); +const OCFLRepository = require('ocfl').Repository; +var argv = require('yargs').argv; +var Utils = require("ro-crate").Utils +var ROCrate = require('ro-crate').ROCrate; +var utils = new Utils(); + + +// This is an asynchronous library so you need to call using await, or use promises + + +async function main() { + // OCFL repo - exsists? + const repoPath = argv.repo ? argv.repo : "ocfl_demo"; + // Make or open a database + // Get or open an OCFL repo + var repo = new OCFLRepository(); + var index = []; + var init = await repo.load(repoPath); + var objects = await repo.objects(); + var p; + for (let o of Object.keys(objects)) { + var object = objects[o]; + const inv = await (object.getInventory()); + var headState = inv.versions[inv.head].state; + for (let hash of Object.keys(headState)){ + if (headState[hash].includes("ro-crate-metadata.jsonld")) { + p = inv.manifest[hash][0]; + break; + } + } + + var rocrateFile = path.join(object.path, p); + console.log("GOT", rocrateFile); + var json = JSON.parse(fs.readFileSync(rocrateFile)); + crate = new ROCrate(json); + crate.index(); + var dataset = crate.getRootDataset() + console.log(dataset['@id'], dataset['description'], p); + var newItem = { + "@id": dataset["@id"], + name: utils.asArray(dataset.name)[0], + description: utils.asArray(dataset.description)[0], + datePublished: utils.asArray(dataset.name)[0], + path: object.path + } + index.push(newItem); + + fs.writeFileSync(path.join(repo.path, "index.json"), JSON.stringify(index, null, 2)); + /* + TODO + + - Find the dataset [] + + - Sumamarize it + + - add to an index object + + - write out the object + + */ + + } +} +main(); + diff --git a/makefile b/makefile new file mode 100644 index 0000000000000000000000000000000000000000..d6bcaed64350b47808944d4ae47b0389f1611955 --- /dev/null +++ b/makefile @@ -0,0 +1,32 @@ +all: sample glop GTM luckett index + +index: + node make-ocfl-index.js --repo $(REPO_PATH) + +sample: + find $(CALCYFY_PATH)/test_data/sample -name ~* -delete + find $(CALCYFY_PATH)/test_data/sample -name .DS_Store -delete + calcyfy -r $(CALCYFY_PATH)/test_data/sample -c https://data.research.uts.edu.au/examples/ro-crate/0.2 + node ro-crate-deposit.js --repo $(REPO_PATH) --from $(CALCYFY_PATH)/test_data/sample + +glop: + find $(CALCYFY_PATH)/test_data/Glop_Pot -name ~* -delete + find $(CALCYFY_PATH)/test_data/Glop_Pot -name .DS_Store -delete + calcyfy -r $(CALCYFY_PATH)/test_data/Glop_Pot -c https://data.research.uts.edu.au/examples/ro-crate/0.2 + node ro-crate-deposit.js --repo ~/working/ocfl-demo --from $(CALCYFY_PATH)/test_data/Glop_Pot + +GTM: + find $(CALCYFY_PATH)/test_data/GTM -name ~* -delete + find $(CALCYFY_PATH)/test_data/GTM -name .DS_Store -delete + calcyfy -r $(CALCYFY_PATH)/test_data/GTM -c https://data.research.uts.edu.au/examples/ro-crate/0.2 + node ro-crate-deposit.js --repo $(REPO_PATH) --from $(CALCYFY_PATH)/test_data/GTM + +luckett: + find $(CALCYFY_PATH)/test_data/luckett -name ~* -delete + find $(CALCYFY_PATH)/test_data/luckett -name .DS_Store -delete + calcyfy -r $(CALCYFY_PATH)/test_data/luckett -c https://data.research.uts.edu.au/examples/ro-crate/0.2 + node ro-crate-deposit.js --repo $(REPO_PATH) --from $(CALCYFY_PATH)/test_data/luckett + + +push: + rsync ~/working/ro-crate/0.2bags/* ptsefton@138.25.37.46:/mnt/datacrateexample/ro-crate/0.2 -ruvi --delete diff --git a/package.json b/package.json new file mode 100644 index 0000000000000000000000000000000000000000..0abea9a3a950f1946caefdae014cd72d189923d3 --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "ocfl-demos", + "version": "1.0.0", + "description": "UTS Specific OCFL demos for ARDC project", + "main": "make_demo_repo.js", + "scripts": { + "test": "mocha" + }, + "repository": { + "type": "git", + "url": "TBA" + }, + "author": "UTS eResearch Staff", + "license": "GPL-3.0", + "dependencies": { + "fs-extra": "^8.1.0", + "ocfl": "^1.0.3", + "ro-crate": "^1.0.1", + "yargs": "^13.3.0" + } +} diff --git a/ro-crate-deposit.js b/ro-crate-deposit.js new file mode 100644 index 0000000000000000000000000000000000000000..2ce5f578c0072ac5354bf39c498d30764d677b86 --- /dev/null +++ b/ro-crate-deposit.js @@ -0,0 +1,33 @@ + +const path = require('path'); +const fs = require('fs-extra'); +const ocfl = require("ocfl"); +console.log(ocfl); +const OCFLRepository = require('ocfl').Repository; +var argv = require('yargs').argv; + + + + +async function main() { + // OCFL repo - exsists? + const fromPath = argv.from ? argv.from : "."; + const repoPath = argv.repo ? argv.repo : "ocfl_demo"; + + // Get or open an OCFL repo + var repo = new OCFLRepository(); + + // TODO - Load as an OCFL object to get ID use instead of fromPath below + + + if (fs.existsSync(repoPath)) { + var init = await repo.load(repoPath); + } else { + fs.mkdirSync(repoPath); + init = await repo.create(repoPath) + } + const new_object1 = await repo.importNewObjectDir(fromPath, fromPath); + console.log(new_object1); +} +main(); +