diff --git a/CATALOG.json b/CATALOG.json
index c7fed48e870f41c00184b2744139cea762d7ad73..be0cf704ff44c692c03966d665b2264a9f36eda0 100644
--- a/CATALOG.json
+++ b/CATALOG.json
@@ -2315,6 +2315,9 @@
{
"@id": "temporal_coverage.png"
},
+ {
+ "@id": "template.html"
+ },
{
"@id": "build"
}
@@ -2373,7 +2376,7 @@
"@type": "File",
"contentSize": "12602",
"path": "bibliography.bib",
- "description": "References for the paper: ",
+ "description": "References for the submission",
"encodingFormat": "Plain Text File",
"fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/x-fmt/111",
"filename": [],
@@ -2387,7 +2390,7 @@
},
"contentSize": "11379",
"path": "bibliography.csl",
- "description": "Citation definition. Modified by Peter Sefton to always show a URL for each reference",
+ "description": "Citation definition. Modified by Peter Sefton to always show a URL for each reference, but needs more work as it is duplicating URLs",
"encodingFormat": "Extensible Markup Language",
"fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/fmt/101",
"filename": [],
@@ -2403,16 +2406,53 @@
"@type": "Dataset",
"path": "build",
"description": "Contains a PDF version of source file",
- "hasPart": {
- "@id": "build/paper.pdf"
- },
+ "hasPart": [
+ {
+ "@id": "build/paper.pdf"
+ },
+ {
+ "@id": "build/paper.html"
+ },
+ {
+ "@id": "build/temporal_coverage.png"
+ }
+ ],
"identifier": "build",
"name": " Directory of automatically created files"
},
+ {
+ "@id": "build/paper.html",
+ "@type": "File",
+ "contentSize": "28571",
+ "path": "build/paper.html",
+ "creator": [
+ {
+ "@id": "Â https://orcid.org/0000-0002-3545-944X"
+ },
+ {
+ "@id": "https://orcid.org/0000-0001-5152-5307"
+ },
+ {
+ "@id": "Â https://orcid.org/0000-0002-0074-112X"
+ },
+ {
+ "@id": "https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM="
+ }
+ ],
+ "description": "HTML Version",
+ "encodesCreativeWork": {
+ "@id": "paper.md"
+ },
+ "encodingFormat": "Hypertext Markup Language",
+ "fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/fmt/96",
+ "filename": [],
+ "license": "https://creativecommons.org/licenses/by/3.0/au/",
+ "name": "DataCrate: a method of packaging, distributing, displaying and archiving Research Objects (HTML)"
+ },
{
"@id": "build/paper.pdf",
"@type": "File",
- "contentSize": "284183",
+ "contentSize": "289278",
"path": "build/paper.pdf",
"creator": [
{
@@ -2423,6 +2463,9 @@
},
{
"@id": "Â https://orcid.org/0000-0002-0074-112X"
+ },
+ {
+ "@id": "https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM="
}
],
"description": "PDF verision of DataCrate: a method of packaging, distributing, displaying and archiving Research Objects",
@@ -2435,6 +2478,18 @@
"license": "https://creativecommons.org/licenses/by/3.0/au/",
"name": "DataCrate: a method of packaging, distributing, displaying and archiving Research Objects (PDF)"
},
+ {
+ "@id": "build/temporal_coverage.png",
+ "@type": "File",
+ "contentSize": "15998",
+ "path": "build/temporal_coverage.png",
+ "encodingFormat": "Portable Network Graphics",
+ "fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/fmt/13",
+ "filename": [],
+ "sameAs": {
+ "@id": "temporal_coverage.png"
+ }
+ },
{
"@id": "http://uts.edu.au",
"@type": "Organization",
@@ -2447,6 +2502,15 @@
"identifier": "http://wsu.edu.au",
"name": "Western Sydney University"
},
+ {
+ "@id": "https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=",
+ "@type": "Person",
+ "email": "duncan.loxton@uts.edu.au",
+ "familyName": "Loxton",
+ "givenName": "Duncan",
+ "identifier": "https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=",
+ "name": "Duncan Loxton"
+ },
{
"@id": "https://github.com/stsewd/ieee-pandoc-template",
"@type": "Project",
@@ -2458,7 +2522,7 @@
"@id": "https://orcid.org/0000-0001-5152-5307",
"@type": "Person",
"affiliation": "University of Technology Sydney",
- "email": "michel.lynch@uts.edu.au",
+ "email": "michael.lynch@uts.edu.au",
"familyName": "Lynch",
"givenName": "Michael",
"identifier": "https://orcid.org/0000-0001-5152-5307",
@@ -2467,7 +2531,7 @@
{
"@id": "makefile",
"@type": "File",
- "contentSize": "279",
+ "contentSize": "411",
"path": "makefile",
"description": "Part of the IEEE Paper Template for Pandoc",
"encodingFormat": "Plain Text File",
@@ -2483,7 +2547,7 @@
{
"@id": "metadata.yaml",
"@type": "File",
- "contentSize": "1411",
+ "contentSize": "1625",
"path": "metadata.yaml",
"creator": [
{
@@ -2494,6 +2558,9 @@
},
{
"@id": "Â https://orcid.org/0000-0002-0074-112X"
+ },
+ {
+ "@id": "https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM="
}
],
"description": "Metadata template from IEEE Paper Template for Pandoc filled out with author names, abstract & keywords",
@@ -2505,9 +2572,9 @@
{
"@id": "paper.md",
"@type": "File",
- "contentSize": "20989",
+ "contentSize": "21054",
"path": "paper.md",
- "description": "Source for the submitted paper: ",
+ "description": "Markdown source file",
"encodingFormat": "Plain Text File",
"fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/x-fmt/111",
"filename": [],
@@ -2517,6 +2584,20 @@
"@id": "paper.md"
}
},
+ {
+ "@id": "template.html",
+ "@type": "File",
+ "contentSize": "607",
+ "path": "template.html",
+ "creator": {
+ "@id": "Â https://orcid.org/0000-0002-3545-944X"
+ },
+ "description": "HTML Template",
+ "encodingFormat": "Hypertext Markup Language",
+ "fileFormat": "http://www.nationalarchives.gov.uk/PRONOM/fmt/96",
+ "filename": [],
+ "license": "https://creativecommons.org/licenses/by/3.0/au/"
+ },
{
"@id": "template.latex",
"@type": "File",
@@ -2553,7 +2634,7 @@
"affiliation": {
"@id": "http://wsu.edu.au"
},
- "email": "gerard.devine@wsu.edu.au",
+ "email": "g.devine@wsu.edu.au",
"familyName": "Devine",
"givenName": "Gerard",
"identifier": "Â https://orcid.org/0000-0002-0074-112X",
diff --git a/CATALOG_datacrate-RO-2018.xlsx b/CATALOG_datacrate-RO-2018.xlsx
index 3560f963fdc6d9f4ede898d495cd4e320720cba6..44c5ad3855717fddc8a4f1f468c94b1ff33b996d 100644
Binary files a/CATALOG_datacrate-RO-2018.xlsx and b/CATALOG_datacrate-RO-2018.xlsx differ
diff --git a/build/CATALOG_build.xlsx b/build/CATALOG_build.xlsx
index 37c32f5384ab174f3b322f4b1188052a9bf43a23..651c01a22f85cc6058204e1eaf3c10e0d4096dff 100644
Binary files a/build/CATALOG_build.xlsx and b/build/CATALOG_build.xlsx differ
diff --git a/build/paper.html b/build/paper.html
new file mode 100644
index 0000000000000000000000000000000000000000..4bb05b58af1c51bf6f1ebd852b7335fee2ef4cbc
--- /dev/null
+++ b/build/paper.html
@@ -0,0 +1,181 @@
+
+<html>
+<head>
+ <link rel="stylesheet"
+ href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"
+ integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u"
+ crossorigin="anonymous"/>
+
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+
+ <title>DataCrate: a method of packaging, distributing, displaying and archiving Research Objects</title>
+</head>
+<body>
+<h1>DataCrate: a method of packaging, distributing, displaying and archiving Research Objects</h1>
+ <ul>
+ <li>Peter Sefton Sydney, Australia peter.sefton@uts.edu.au</li>
+
+ <li>Michael Lynch Sydney, Australia michael.lynch@uts.edu.au</li>
+
+ <li>Gerard Devine Sydney, Australia g.devine@wsu.edu.au</li>
+
+ <li>Duncan Loxton Sydney, Australia duncan.loxton@uts.edu.au</li>
+
+ </ul>
+
+ <h1 id="note-to-reviewers">Note to reviewers</h1>
+ <p>The specification described here is currently in draft at v0.2. A version one release is planned for October 2018. If accepted this extended abstract would be updated for presentation and subsequent publication.</p>
+ <h1 id="introduction">Introduction</h1>
+ <p>In illuminating the term <em>Research Object</em> the call for proposals for Research Object 2018 uses the phrase “multi-part research outcomes with their contextâ€. The DataCrate specification is a research data packaging and dissemination specification designed to capture exactly that; outcomes (also inputs) and context.</p>
+ <p>DataCrate specifies how to gather-together data in such a way that it can (a) be packaged via Zip, tar, a disc image, a multi-part package or (b) be hosted on a web server or file share for inspection by potential users and/or used directly on High Performance Computing systems or otherwise accessed and analysed.</p>
+ <p>DataCrates can contain any kind of data, and the context can include, but is not limited to, data about the people, software and equipment used in the research as well as supporting documents such as publications, funding agreements or README files.</p>
+ <h1 id="methodology">Methodology</h1>
+ <p>The DataCrate specification grew out of two generations of previous data packaging work. The first implementation was in the <a href="https://www.westernsydney.edu.au/eresearch/home/projects/completed_projects/hiev">HIEv system at Western Sydney University</a>, a data capture system for environmental science data that captures data files produced by sensor networks and allows manual or API-based upload of other files. Using a web interface researchers can select files to export, for example to support an research article.</p>
+ <p>The requirements / principles for the HIEv system were presented at eResearch Australasia <span class="citation" data-cites="seftonIntroducingNextYear2013">[1]</span>.</p>
+ <blockquote>
+ <ol type="1">
+ <li>The packaging format should not be data-format-sensitive</li>
+ <li>The packaging format should not be research domain specific</li>
+ <li>The packaging format should not be technology or platform specific</li>
+ <li>The data package should contain as much contextual information as possible</li>
+ <li>Metadata should be easily human and machine-readable</li>
+ <li>The package format should contain self-checking and verification features</li>
+ <li>The metadata format should be compatible with the semantic web by using URIs as names for things incuding metadata terms*</li>
+ <li>Requirement <code>7</code> implies using Linked-Data <span class="citation" data-cites="berners-leeLinkedData2006">[2]</span>, but the project should not attempt to define and manage its own ontologies, for reasons of sustainability*</li>
+ <li>A data package should be able to be displayed on the web - implying that the human readable metadata in <code>5</code> should be in HTML.*</li>
+ </ol>
+ </blockquote>
+ <p>*The last three requirements or principles were not explicit in the presentation but were discussed during the development, and proved increasingly important in the development of the current DataCrate specification.</p>
+ <p>The data packaging in HIEv used the Bagit <span class="citation" data-cites="kunzeBagItFilePackaging">[3]</span> packaging spec to cover requirement <code>6</code> - BagIt also doesn’t get in the way of any of the other requirements.</p>
+ <p>The main innovation in HIEv’s packaging was to add an HTML file that covered requirements <code>4</code> (as much context as possible) & <code>5</code> (human and machine readable metadata). To do this, HIEv produces a summary of the context, with information about the facilities used – their name, nature and location – and technical details about the payload files, thus satisfying <code>5</code>, using RDFa <span class="citation" data-cites="RDFa2014">[4]</span> to embed metadata in the HTML file gave both human (HTML) and machine (RDFa) views of the data.</p>
+ <p><a href="https://www.westernsydney.edu.au/eresearch/home/projects/cr8it">Cr8it</a> <span class="citation" data-cites="seftonPickPackagePublish2014">[5]</span> was another early implementation that used the same basic idea for data packages in the OwnCloud file sharing application.</p>
+ <p>The first two DataCrate proto-implementations had no guidelines for what metadata to use beyond what was hard-wired into each code-base, so there was no hope of easy interoperability or safe extensibility, and there were no repositories into which data could be published, but we had good feedback about the concept from the eResearch community and from the very limited number of researchers exposed to the systems, so in 2016 when UTS began work on a new Research Data Management service <span class="citation" data-cites="wheelerEndtoEndResearchData2018">[6]</span>, we decided to properly specify a data packaging format that met the above requirements and the DataCrate standard was born.</p>
+ <p>A team based at UTS, with some external collaborators started a process to work out (a) was there an existing standard which had emerged since the HIEv work that met the requirements? (b) If no, which RDF vocabularies should we use? and (c) the mechanics of organising the files in the packages. At this point the requirements had evolved to be:</p>
+ <p>We were looking to find or build a data packaging format which had the following di:</p>
+ <ol type="1">
+ <li>Checksums per-file and the ability to include linked resources (features of BagIt)</li>
+ <li>Linked-data metadata in JSON-LD format using well-documented ontologies / vocabularies with coverage for:
+ <ul>
+ <li>Discovery and citation metadata; who created it, what is it, where did the work take place, what in the world is it <em>about</em>. And, the same metadata at the file level.</li>
+ <li>Technical metadata, what size is each file, what format is the file in, who or what created the file.</li>
+ </ul></li>
+ <li>A convention for including an HTML file which describes the dataset, and potentially all of its files with a human-readable view of <code>2.</code>.</li>
+ </ol>
+ <p>These could all be accomplished by an update of the HIEV data package but it was important to make sure we were not re-inventing something that had been done elsewhere.</p>
+ <h3 id="existing-standards">Existing standards</h3>
+ <p>We were not able to find any general-purpose packaging specification with anything like the HTML+RDFa index that HIEv data packages have, allowing for human and machine readable metadata. Using BagIt plus extra files worked well in our initial implementations so that was to be kept unless a better alternative surfaced – the decisions were around formalising metadata standards.</p>
+ <p>BagIt, which had been used in HIEv and Cr8it, is an obvious standard on which to base a research data packaging format - it is widely used in the research data community, there is cross-platform (requirement <code>3</code>) <a href="https://en.wikipedia.org/wiki/BagIt#Tools">tooling available</a> and it covers the integrity aspects of packaging data.</p>
+ <h3 id="alternatives-considered">Alternatives considered</h3>
+ <p>Frictionless Data Packages [TODO ref], which uses a simple JSON format as a manifest, has roughly equivalent packaging features to BagIt, having checksum features built in. In their favour, Frictionless Data Packages have the ability to describe the headers in tabular data files. However, they do not meet the requirement <code>7</code> of having linked-data metadata, so while the JSON metadata is technically machine readable, in that is simple to parse, it is not easy to relate to the semantic web as it does not use linked-data standards, and the terms are defined locally to the specification, without URIs. It is also unclear how to extend the specification in a standardised way, contrasting with linked-data approaches which <em>automatically</em> allow extension by the use of URIs.</p>
+ <p>As an example, <a href="https://frictionlessdata.io/specs/data-package/">the spec</a> does not give a single way to describe temporal coverage</p>
+ <blockquote>
+ <p>Adherence to the specification does not imply that additional, non-specified properties cannot be used: a descriptor MAY include any number of properties in additional to those described as required and optional properties. For example, if you were storing time series data and wanted to list the temporal coverage of the data in the Data Package you could add a property temporal (cf Dublin Core):</p>
+ <p>“temporalâ€: { “nameâ€: “19th Centuryâ€, “startâ€: “1800-01-01â€, “endâ€: “1899-12-31†}</p>
+ <p>This flexibility enables specific communities to extend Data Packages as appropriate for the data they manage. As an example, the Tabular Data Package specification extends Data Package to the case where all the data is tabular and stored in CSV.</p>
+ </blockquote>
+ <blockquote>
+ <p>This flexibility enables specific communities to extend Data Packages as appropriate for the data they manage. As an example, the Tabular Data Package specification extends Data Package to the case where all the data is tabular and stored in CSV. <a href="https://github.com/frictionlessdata/specs/blob/0860ecd6bbb7685425e6493165c9b1a1c91eb16b/specs/data-package.md" class="uri">https://github.com/frictionlessdata/specs/blob/0860ecd6bbb7685425e6493165c9b1a1c91eb16b/specs/data-package.md</a></p>
+ </blockquote>
+ <p>We think that this <em>laissez faire</em> extension mechanism in Frictionless Data Packages is likely to result in a proliferation of highly divergent non-standardised metadata - by using JSON-LD and specifying how to represent temporal and geographical coverage, etc DataCrate aims to encourage common behaviours, to enable interchange of metadata. In DataCrate, the approach is to use schema.org’s temporalCoverage property. Here is an <a href="https://github.com/UTS-eResearch/datacrate/blob/22aebdcd179cb3f9b8141ca350ffafa202f5b523/spec/0.2/data_crate_specification_v0.2.md">example from v0.2</a> of the specification.</p>
+ <blockquote>
+ <p>{ “@idâ€: “https://doi.org/10.5281/zenodo.1009240â€, “@typeâ€: “Datasetâ€, <…> “nameâ€: “Sample dataset for DataCrate v0.2â€, “publisherâ€: { “@idâ€: “http://uts.edu.au†}, “temporalCoverageâ€: “2017†},</p>
+ </blockquote>
+ <p>In the <a href="https://github.com/UTS-eResearch/datacrate/blob/master/spec/0.2/context.json">DataCrate JSON-LD context</a> this expands to URI: <a href="https://schema.org/temporalCoverage" class="uri">https://schema.org/temporalCoverage</a>. And, in the HTML file that displays metadata for humans, the HTML is self-documenting:</p>
+ <figure>
+ <img src="temporal_coverage.png" alt="Screenshot showing how the term “temporalCoverage is linked - the link resolves to the schema.org page for temporalCoverageâ€" /><figcaption>Screenshot showing how the term “temporalCoverage is linked - the link resolves to the schema.org page for temporalCoverageâ€</figcaption>
+ </figure>
+ <p>The ability to resolve URIs is an important one if the <code>index.html</code> file is to be useful to humans. Many URIs in scientific ontologies do not resolve to a web-page but to an ontology file, which is unhelpful for people trying to understand what metadata terms <em>mean</em>.</p>
+ <p>The other main alternative was the Research Object Bundle specification <span class="citation" data-cites="soiland-reyesResearchObjectBundle2014">[7]</span>.</p>
+ <p>Rather than BagIt, the original version of Research Object Bundle uses the Zip-based Universal Container Format - a format for which the documentation now seems to be unavailable from Adobe and which does not have integrity features such as checksums but there is <a href="https://github.com/ResearchObject/bagit-ro">a version of Research Object which uses BagIt</a>.</p>
+ <p>RO BagIt <em>does</em> use Linked-Data and for that reason was given careful consideration as a base-format for DataCrate. However, there were some implementation details that we thought would make it hard for tool-makers (including the core team at UTS); the use of “aggregations†and “annotations†introduces two extra layers of abstraction for describing resources.</p>
+ <p>For example, using this <a href="https://github.com/ResearchObject/bagit-ro/blob/f5fca3abad60c86b3c4f95948b5d64c3bc8e51c6/example1/metadata/manifest.json">sample from the bagit-ro tool</a></p>
+ <p>There is a section in the manifest that lists aggregated files:</p>
+ <blockquote>
+ <p>“aggregatesâ€: [</p>
+ <p>{ “uriâ€: “../data/numbers.csvâ€, “mediatypeâ€: “text/csv†},</p>
+ </blockquote>
+ <p>And a separate place to describe <em>annotations</em> on those files:</p>
+ <blockquote>
+ <p>“annotationsâ€: [</p>
+ <p>{ “aboutâ€: “../data/numbers.csvâ€, “contentâ€: “annotations/numbers.jsonldâ€, “createdByâ€: { “nameâ€: “Stian Soiland-Reyesâ€, “orcidâ€: “http://orcid.org/0000-0001-9842-9718†} }</p>
+ </blockquote>
+ <p>With the actual description of the numbers.csv file residing in <code>annotations/numbers.jsonld</code>.</p>
+ <blockquote>
+ <p>{ “@contextâ€: { “@vocabâ€: “http://purl.org/dc/terms/â€, “dcmiâ€: “http://purl.org/dc/dcmitype/Datasetâ€}, “@idâ€: “../../data/numbers.csvâ€, “@typeâ€: “dcmi:Datasetâ€, “titleâ€: “CSV files of beverage consumptionâ€, “descriptionâ€: “A CSV file listing the number of cups/mugs consumed per person.†}</p>
+ </blockquote>
+ <h1 id="implementation-of-datacrate">Implementation of DataCrate</h1>
+ <p>In our judgement, the level of indirection and number of files involved in the Research Object approach were not suitable for DataCrate; the implementation cost for tool makers would be too high. In making this choice we forewent the benefits of being able to make assertions about the provenance of annotations as distinct resources, and the more intellectually satisfying abstractions about aggregations offered by ORE. We settled on a an approach which used just three extra files:</p>
+ <ol type="1">
+ <li>A single CATALOG.json file, containing JSON-LD which describes the folder/file hierarchy of the data crate and associated contextually relevant entities, such as people in one place.</li>
+ <li>An index.html file with a human-readable summary of the catalog file.</li>
+ <li>Optionally, a <code>DataCite.xml</code> file containing a data citation (a text version of which is prominent in the HTML file if it exists).</li>
+ </ol>
+ <p>Uses for DataCrates planned at UTS include: - Making DataCrates available for download from a public website, both packaged as Zip files and expanded so that users can peruse the <code>index.html</code> file and access individual files. - Using the DataCrate with additional metadata for archiving and preserving data (in a project to begin in 2019) - Using the DataCrate format to allow exchange of data between systems, for example sending data from a repository in a university facility such as the Omero microscopy repository <span class="citation" data-cites="OMEROFlexibleModeldriven">[8]</span> to a git project system like GitLab. - Automatically detecting metadata in DataCrates which are uploaded to our research-integrity driven Research Data Management system <span class="citation" data-cites="wheelerEndtoEndResearchData2018">[6]</span>.</p>
+ <p>The initial version of DataCrate (v0.1) was developed in 2017. V0.1 persisted with HTML+RDFa for human and machine readability but this was cumbersome to generate and was removed at <a href="https://github.com/UTS-eResearch/datacrate/issues/14">the suggestion of Eoghan Ó Carragáin</a> in favour of an approach where the human-centred HTML page is generated from a machine-readable JSON-LD file rather than the other way around.</p>
+ <p>We looked at a variety of standards, including Dublin Core <span class="citation" data-cites="kunzeDublinCoreMetadata2007">[9]</span> which is very limited in coverage and DCAT <span class="citation" data-cites="maaliDataCatalogVocabulary2014">[10]</span> which is more complete for describing data sets at the top level, but silent on the issue of describing files or other contextual entities and relationships betwteen them. We discovered that Schema.org has the widest range of terms needed to describe “who, what, where†metadata for datasets.</p>
+ <p>The DataCrate spec recommend other ontologies where schema.org has gaps in its coverage:</p>
+ <ul>
+ <li><p>For describing datasets such as exported content from digital object repository systems DataCrate uses the Portland Common Data Model <span class="citation" data-cites="PortlandCommonData">[11]</span>, (PCDM), which is a simple ontology for describing nested <a href="http://pcdm.org/models#Collection">Collections</a> of <a href="https://pcdm.org/2016/04/18/models#Object">Objects</a>, with Objects <a href="http://pcdm.org/models#hasFile">having</a> <a href="http://pcdm.org/models#File">Files</a>.</p></li>
+ <li><p>For concepts related to the scholarly process (but biased towards publishing) DataCrate uses terms from the SPAR ontologies <span class="citation" data-cites="shottonIntroductionSemanticPublishing2010">[12]</span> for scholarly communications. For example, schema.org does not have a class for Project so the key “Project†is mapped to the frapo term <a href="https://sparontologies.github.io/frapo/current/frapo.html#d4e2428">Project</a></p></li>
+ </ul>
+ <p>NOTE: we are in the process of adding more coverage for provenance - showing how files are created by software etc. Will be asking the Research Object team for assistance with this.</p>
+ <h1 id="tools">Tools</h1>
+ <p>There are a number of tools for DataCrate in development.</p>
+ <p>At the University of Technology Sydney <span class="citation" data-cites="wheelerEndtoEndResearchData2018">[6]</span>, the Provisioner is an open framework for integrating good research data management practices into everyday research workflows. It provides a user-facing research data management planning tool which allows researchers to describe and publish datasets and create and share workspaces in different research apps such as lab notebooks, code repositories (where data is included by-reference), survey tools and collection management tools. DataCrates are used as an interchange format to move data between the different research apps, and as an ingest, archive and publication format. Lightweight adaptors coded against each research app’s native API allow export and import of DataCrates, which are then used to move data from one app to another, while recording a provenance history in the DataCrates’ metadata. Examples of DataCrates moving through the research lifecycle will be provided.</p>
+ <p>HIEv DataCrate - At the Hawkesbury Institute for the Environment at Western Sydney University, HIEV harvests a wide range of environmental data (and associated file level metadata) from both automated sensor networks and analysed datasets generated by researchers. Leveraging built-in APIs within the HIEv a new packaging function has been developed, allowing for selected datasets to be identified and packaged in the DataCrate standard, complete with metadata automatically exported from the HIEv metadata holdings into the JSON-LD format. Going forward this will allow datasets within HIEv to be published regularly and in an automated fashion, in a format that will increase their potential for reuse.</p>
+ <p><a href="https://code.research.uts.edu.au/eresearch/CalcyteJS">Calcytejs</a> is a command line tool for packaging data into DataCrate developed at the University of Technology Sydney which allows researchers to describe any data set via the use of spreadsheets which the tool auto-creates in a directory tree.</p>
+ <p><a href="https://github.com/UTS-eResearch/omeka-datacrate-tools">Omeka DataCrate Tools</a> is Python tool in early development to export data from Omeka Classic repositories into the DataCrate format.</p>
+ <p>NOTE: A tool currently in development for exporting DataCrates from the Omero microscopy repository will also be presented, and there has been some interest from Caltech in adding DataCrate support to their <a href="http://caltechlibrary.github.io/dataset">dataset</a> project.</p>
+ <h1 id="conclusion">Conclusion</h1>
+ <p>DataCrate (which will be in version 1 by the time of the Research Object workshop) has been tested on a variety of research data sets. Some examples are:</p>
+ <ul>
+ <li><p>Data relating to the IDRC funded project (described in https://doi.org/10.3897/rio.2.e8880) <a href="https://data.research.uts.edu.au/examples/v0.2/Data_Package-IDRC_Opportunities_and_Challenges_Open_Research_Strategies/">to examine data management policies and implementation for development funders</a>. The project involved two parts: a review based on desk work and expert interviews and seven case studies of existing IDRC-funded projects. The case studies were supported by an Introductory Workshop in which the idea of data was examined and the issues involved in sharing discussed in detail. This was followed by an implementation phase in which the projects were supported in developing Data Management Plans. The performance against those plans was then assessed both by the participants and as part of the overall project to generate case studies that are to be published as part of the related RIO Journal Collection. The final project report will also be part of the same collection.</p></li>
+ <li><p>Some <a href="https://data.research.uts.edu.au/examples/v0.2/GTM/">Matlab code</a> that supports a research article. This is a good illustration of how common names (like Lu, J) can be disambiguated using Orcid identifiers.</p></li>
+ <li><p>A small <a href="https://data.research.uts.edu.au/examples/v0.2/Glop_Pot/">archival collection of speleological (cave) mapping data</a>.</p></li>
+ <li><p>Another cave dataset. This one is a <a href="https://data.research.uts.edu.au/examples/v0.2/Victoria_Arch_pub/">3d survey conducted using a lidar scanner mounted on a drone</a>.</p></li>
+ <li><p>A <a href="https://data.research.uts.edu.au/examples/v0.2/sample/">sample data set with one picture in it</a>.</p></li>
+ <li><p>Some <a href="https://data.research.uts.edu.au/examples/v0.2/timluckett/">clinical trial data</a> - this dataset shows how researcher affiliations can be modelled using linked data.</p></li>
+ </ul>
+ <p>At the time of the workshop, we will be in a position demonstrate a number of tools using DataCrate including a live data-publishing workflow implemented at UTS.</p>
+ <p>Future plans for DataCrate include: - More domain specific metadata including scientific properties such as temperature - More use-cases including preservation - Describing the contents of files, such as column headers in tabular data formats.</p>
+ <p>Some issues that would be useful to discuss at the workshop are: - Difficulties in finding good JSON-LD-ready ontologies for research domains. (Should we be working on a schema.org extension for research?) - A lack of simple to use JSON-LD tools for programers (at least in Javascript and Python), we have not been able to find simple to use tools to load a JSON-LD file and traverse it in code.</p>
+ <h1 id="references" class="unnumbered">References</h1>
+ <div id="refs" class="references">
+ <div id="ref-seftonIntroducingNextYear2013">
+ <p>[1] P. Sefton and P. Bugeia, “Introducing next year’s model, the data-crate; applied standards for data-set packaging,†in <em>eResearch Australasia 2013</em>, 2013[Online]. Available: <a href="http://eresearchau.files.wordpress.com/2013/08/eresau2013_submission_57.pdf" class="uri">http://eresearchau.files.wordpress.com/2013/08/eresau2013_submission_57.pdf</a>.</p>
+ </div>
+ <div id="ref-berners-leeLinkedData2006">
+ <p>[2] T. Berners-Lee, <em>Linked data, 2006</em>. [Online]. Available: <a href="http://www.w3.org/DesignIssues/LinkedData.html" class="uri">http://www.w3.org/DesignIssues/LinkedData.html</a>.</p>
+ </div>
+ <div id="ref-kunzeBagItFilePackaging">
+ <p>[3] J. Kunze, A. Boyko, B. Vargas, L. Madden, and J. Littman, “The BagIt File Packaging Format (V0.97).†http://tools.ietf.org/html/draft-kunze-bagit-06[Online]. Available: <a href="http://tools.ietf.org/html/draft-kunze-bagit-06" class="uri">http://tools.ietf.org/html/draft-kunze-bagit-06</a>. [Accessed: 01-Mar-2013].</p>
+ </div>
+ <div id="ref-RDFa2014">
+ <p>[4] “RDFa,†<em>Wikipedia, the free encyclopedia</em>, Feb. 2014[Online]. Available: <a href="http://en.wikipedia.org/w/index.php?title=RDFa\\\&oldid=592600280" class="uri">http://en.wikipedia.org/w/index.php?title=RDFa\\\&oldid=592600280</a>. [Accessed: 26-Feb-2014].</p>
+ </div>
+ <div id="ref-seftonPickPackagePublish2014">
+ <p>[5] P. Sefton, P. Bugeia, and V. Picasso, “Pick, Package and Publish research data: Cr8it and Of The Web,†in <em>eResearch Australasia 2014</em>, 2014[Online]. Available: <a href="http://eresearchau.files.wordpress.com/2014/07/eresau2014_submission_30.pdf" class="uri">http://eresearchau.files.wordpress.com/2014/07/eresau2014_submission_30.pdf</a>.</p>
+ </div>
+ <div id="ref-wheelerEndtoEndResearchData2018">
+ <p>[6] L. Wheeler, S. Wise, and P. Sefton, “End-to-End Research Data Management for the Responsible Conduct of Research at the University of Technology Sydney,†2018[Online]. Available: <a href="https://eresearch.uts.edu.au/2018/07/04/APRI_2018_provisioner.htm" class="uri">https://eresearch.uts.edu.au/2018/07/04/APRI_2018_provisioner.htm</a>. [Accessed: 10-Jul-2018].</p>
+ </div>
+ <div id="ref-soiland-reyesResearchObjectBundle2014">
+ <p>[7] S. Soiland-Reyes, M. Gamble, and R. Haines, “Research object bundle 1.0,†<em>Specification, researchobject. org</em>, 2014.</p>
+ </div>
+ <div id="ref-OMEROFlexibleModeldriven">
+ <p>[8] “OMERO: Flexible, model-driven data management for experimental biology | Nature Methods.†https://www.nature.com/articles/nmeth.1896[Online]. Available: <a href="https://www.nature.com/articles/nmeth.1896" class="uri">https://www.nature.com/articles/nmeth.1896</a>. [Accessed: 13-Jul-2018].</p>
+ </div>
+ <div id="ref-kunzeDublinCoreMetadata2007">
+ <p>[9] J. Kunze and T. Baker, “The Dublin Core Metadata Element Set,†2007[Online]. Available: <a href="http://www.rfc-editor.org/info/rfc5013" class="uri">http://www.rfc-editor.org/info/rfc5013</a>. [Accessed: 12-Jul-2018].</p>
+ </div>
+ <div id="ref-maaliDataCatalogVocabulary2014">
+ <p>[10] F. Maali, J. Erickson, and P. Archer, “Data catalog vocabulary (DCAT),†<em>W3C Recommendation</em>, vol. 16, 2014[Online]. Available: <a href="https://www.w3.org/TR/2014/REC-vocab-dcat-20140116/" class="uri">https://www.w3.org/TR/2014/REC-vocab-dcat-20140116/</a>.</p>
+ </div>
+ <div id="ref-PortlandCommonData">
+ <p>[11] “Portland Common Data Model.†https://pcdm.org/2016/04/18/models[Online]. Available: <a href="https://pcdm.org/2016/04/18/models" class="uri">https://pcdm.org/2016/04/18/models</a>. [Accessed: 12-Jul-2018].</p>
+ </div>
+ <div id="ref-shottonIntroductionSemanticPublishing2010">
+ <p>[12] D. Shotton, “Introduction the Semantic Publishing and Referencing (SPAR) Ontologies. October 14, 2010,†<em>URL: http://opencitations. wordpress. com/2010/10/14/introducing-thesemantic-publishingand-referencing-spar-ontologies</em>, 2010.</p>
+ </div>
+ </div>
+</body>
diff --git a/build/paper.pdf b/build/paper.pdf
index a6cb5d175c6ca2a2fef52f2bf8686a3b2bcf3576..46bc71a4036f9fde112872273b981af1cfaa47d2 100644
Binary files a/build/paper.pdf and b/build/paper.pdf differ
diff --git a/build/temporal_coverage.png b/build/temporal_coverage.png
new file mode 100644
index 0000000000000000000000000000000000000000..1cfb9a3574a670a6367b472e1f514f9a9aa0f743
Binary files /dev/null and b/build/temporal_coverage.png differ
diff --git a/build/~$CATALOG_build.xlsx b/build/~$CATALOG_build.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..57963342946617ec0969c815d5393f28eda62c45
Binary files /dev/null and b/build/~$CATALOG_build.xlsx differ
diff --git a/index.html b/index.html
index 2d5e48c086692b7d67d7f3017afd0d9ae2a93cf2..15d94a1df7386d56fe5ffeca2163c6610632f3a2 100644
--- a/index.html
+++ b/index.html
@@ -127,6 +127,8 @@ A machine-readable version of this page is available:
,
<a href="#temporal_coverage.png">temporal_coverage.png</a>
,
+ <a href="#template.html">template.html</a>
+ ,
<a href="#build"> Directory of automatically created files</a>
</td>
</tr>
@@ -309,7 +311,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/description">description</a>
</th>
- <td>References for the paper: </td>
+ <td>References for the submission</td>
</tr>
<tr>
<th width="20%">
@@ -353,7 +355,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/description">description</a>
</th>
- <td>Citation definition. Modified by Peter Sefton to always show a URL for each reference</td>
+ <td>Citation definition. Modified by Peter Sefton to always show a URL for each reference, but needs more work as it is duplicating URLs</td>
</tr>
<tr>
<th width="20%">
@@ -433,7 +435,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
</th>
- <td>279</td>
+ <td>411</td>
</tr>
<tr>
<th width="20%">
@@ -475,6 +477,8 @@ A machine-readable version of this page is available:
<a href="#https://orcid.org/0000-0001-5152-5307">Michael Lynch</a>
,
<a href="#Â https://orcid.org/0000-0002-0074-112X">Gerard Devine</a>
+ ,
+ <a href="#https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=">Duncan Loxton</a>
</td>
</tr>
<tr>
@@ -497,7 +501,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
</th>
- <td>1411</td>
+ <td>1625</td>
</tr>
<tr>
<th width="20%">
@@ -525,7 +529,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/description">description</a>
</th>
- <td>Source for the submitted paper: </td>
+ <td>Markdown source file</td>
</tr>
<tr>
<th width="20%">
@@ -547,7 +551,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
</th>
- <td>20989</td>
+ <td>21054</td>
</tr>
<tr>
<th width="20%">
@@ -665,6 +669,58 @@ A machine-readable version of this page is available:
</tr>
</table>
<hr></hr>
+ <table class="table" id="template.html">
+ <tr></tr>
+ <tr>
+ <th width="20%">@type</th>
+ <td>
+ <a class="fa fa-external-link" href="http://schema.org/MediaObject">File</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/description">description</a>
+ </th>
+ <td>HTML Template</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/creator">creator</a>
+ </th>
+ <td>
+ <a href="#Â https://orcid.org/0000-0002-3545-944X">Peter Sefton</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentUrl">path</a>
+ </th>
+ <td>
+ <a href="template.html">template.html</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/encodingFormat">encodingFormat</a>
+ </th>
+ <td>
+ <a href="http://www.nationalarchives.gov.uk/PRONOM/fmt/96" class="fa fa-external-link" title="http://www.nationalarchives.gov.uk/PRONOM/fmt/96">Hypertext Markup Language</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
+ </th>
+ <td>607</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/license">license</a>
+ </th>
+ <td>https://creativecommons.org/licenses/by/3.0/au/</td>
+ </tr>
+ </table>
+ <hr></hr>
<table class="table" id="build">
<tr></tr>
<tr>
@@ -699,6 +755,10 @@ A machine-readable version of this page is available:
</th>
<td>
<a href="#build/paper.pdf">DataCrate: a method of packaging, distributing, displaying and archiving Research Objects (PDF)</a>
+ ,
+ <a href="#build/paper.html">DataCrate: a method of packaging, distributing, displaying and archiving Research Objects (HTML)</a>
+ ,
+ <a href="#build/temporal_coverage.png">build/temporal_coverage.png</a>
</td>
</tr>
</table>
@@ -734,6 +794,8 @@ A machine-readable version of this page is available:
<a href="#https://orcid.org/0000-0001-5152-5307">Michael Lynch</a>
,
<a href="#Â https://orcid.org/0000-0002-0074-112X">Gerard Devine</a>
+ ,
+ <a href="#https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=">Duncan Loxton</a>
</td>
</tr>
<tr>
@@ -756,7 +818,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
</th>
- <td>284183</td>
+ <td>289278</td>
</tr>
<tr>
<th width="20%">
@@ -773,6 +835,118 @@ A machine-readable version of this page is available:
<td>https://creativecommons.org/licenses/by/3.0/au/</td>
</tr>
</table>
+ <hr></hr>
+ <table class="table" id="build/paper.html">
+ <tr></tr>
+ <tr>
+ <th width="20%">@type</th>
+ <td>
+ <a class="fa fa-external-link" href="http://schema.org/MediaObject">File</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/name">name</a>
+ </th>
+ <td>DataCrate: a method of packaging, distributing, displaying and archiving Research Objects (HTML)</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/description">description</a>
+ </th>
+ <td>HTML Version</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/creator">creator</a>
+ </th>
+ <td>
+ <a href="#Â https://orcid.org/0000-0002-3545-944X">Peter Sefton</a>
+ ,
+ <a href="#https://orcid.org/0000-0001-5152-5307">Michael Lynch</a>
+ ,
+ <a href="#Â https://orcid.org/0000-0002-0074-112X">Gerard Devine</a>
+ ,
+ <a href="#https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=">Duncan Loxton</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentUrl">path</a>
+ </th>
+ <td>
+ <a href="build/paper.html">paper.html</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/encodingFormat">encodingFormat</a>
+ </th>
+ <td>
+ <a href="http://www.nationalarchives.gov.uk/PRONOM/fmt/96" class="fa fa-external-link" title="http://www.nationalarchives.gov.uk/PRONOM/fmt/96">Hypertext Markup Language</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
+ </th>
+ <td>28571</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/encodesCreativeWork">encodesCreativeWork</a>
+ </th>
+ <td>
+ <a href="#paper.md">DataCrate: a method of packaging, distributing, displaying and archiving Research Objects</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/license">license</a>
+ </th>
+ <td>https://creativecommons.org/licenses/by/3.0/au/</td>
+ </tr>
+ </table>
+ <hr></hr>
+ <table class="table" id="build/temporal_coverage.png">
+ <tr></tr>
+ <tr>
+ <th width="20%">@type</th>
+ <td>
+ <a class="fa fa-external-link" href="http://schema.org/MediaObject">File</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentUrl">path</a>
+ </th>
+ <td>
+ <a href="build/temporal_coverage.png">temporal_coverage.png</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/encodingFormat">encodingFormat</a>
+ </th>
+ <td>
+ <a href="http://www.nationalarchives.gov.uk/PRONOM/fmt/13" class="fa fa-external-link" title="http://www.nationalarchives.gov.uk/PRONOM/fmt/13">Portable Network Graphics</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/contentSize">contentSize</a>
+ </th>
+ <td>15998</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/sameAs">sameAs</a>
+ </th>
+ <td>
+ <a href="#temporal_coverage.png">temporal_coverage.png</a>
+ </td>
+ </tr>
+ </table>
<h1>
Contextual info:
<span>
@@ -822,6 +996,42 @@ A machine-readable version of this page is available:
</span>
</h1>
<hr></hr>
+ <table class="table" id="https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=">
+ <tr></tr>
+ <tr>
+ <th width="20%">@type</th>
+ <td>
+ <a class="fa fa-external-link" href="http://schema.org/Person">Person</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/name">name</a>
+ </th>
+ <td>
+ <a href="https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=" class="fa fa-external-link" title="https://email.itd.uts.edu.au/webapps/directory/entry.mason?entry=dWlkPTExODAyNSxvdT1TdGFmZixvPXV0cy5lZHUuYXUsbz1VVFM=">Duncan Loxton</a>
+ </td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/email">email</a>
+ </th>
+ <td>duncan.loxton@uts.edu.au</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/familyName">familyName</a>
+ </th>
+ <td>Loxton</td>
+ </tr>
+ <tr>
+ <th width="20%">
+ <a class="fa fa-external-link" href="http://schema.org/givenName">givenName</a>
+ </th>
+ <td>Duncan</td>
+ </tr>
+ </table>
+ <hr></hr>
<table class="table" id="https://orcid.org/0000-0001-5152-5307">
<tr></tr>
<tr>
@@ -848,7 +1058,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/email">email</a>
</th>
- <td>michel.lynch@uts.edu.au</td>
+ <td>michael.lynch@uts.edu.au</td>
</tr>
<tr>
<th width="20%">
@@ -890,7 +1100,7 @@ A machine-readable version of this page is available:
<th width="20%">
<a class="fa fa-external-link" href="http://schema.org/email">email</a>
</th>
- <td>gerard.devine@wsu.edu.au</td>
+ <td>g.devine@wsu.edu.au</td>
</tr>
<tr>
<th width="20%">
diff --git a/makefile b/makefile
index 55c1be655c3b069f58dff6dff121dc237fcfdea9..25cbfd45a4dc8cdfaeba64642862c32eea71e98c 100644
--- a/makefile
+++ b/makefile
@@ -10,10 +10,15 @@ FLAGS = --bibliography=bibliography.bib \
FLAGS_PDF = --template=template.latex
-all: pdf
+FLAGS_HTML = --template=template.html
+
+all: pdf html
pdf:
pandoc -o $(OUTPUT)/paper.pdf $(FLAGS) $(FLAGS_PDF) $(FILES)
+html:
+ pandoc -o $(OUTPUT)/paper.html $(FLAGS) $(FLAGS_HTML) $(FILES)
+ cp *.png build/
clean:
rm build/*
diff --git a/metadata.yaml b/metadata.yaml
index 3c06844ea7835e4be616639992b9152b161c75fd..19e9f4f07393625ad2fbe722b0d5144e09d3d5ae 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -1,5 +1,5 @@
---
-title: 'DataCrate'
+title: 'DataCrate: a method of packaging, distributing, displaying and archiving Research Objects'
author:
- name: Peter Sefton
affilation: University of Technology Sydney
@@ -9,10 +9,15 @@ author:
affilation: University of Technology Sydney
location: Sydney, Australia
email: michael.lynch@uts.edu.au
- - name: Gerrard Devine
+ - name: Gerard Devine
affilation: Western Sydney University
location: Sydney, Australia
- email: gerrard.devine@wsu.edu.au
+ email: g.devine@wsu.edu.au
+
+ - name: Duncan Loxton
+ affilation: University of Technology Sydney
+ location: Sydney, Australia
+ email: duncan.loxton@uts.edu.au
keywords:
- Research Objects
- Data Packaging
diff --git a/paper.md b/paper.md
index 2697432baa4ccf724770ece32e7a0f6d110fcc56..9ee59820cf4715b7ecc755af4031de0320628ad4 100644
--- a/paper.md
+++ b/paper.md
@@ -1,28 +1,8 @@
-# Note to authors
-
-This is a paper submission for [Research Object
-2018](http://www.researchobject.org/ro2018/) - due 2018-07-15 (Monday our time
-should be OK for final submission). If you'd like to add to this, sign in with
-your UTS credentials and change away, but let me know first so we can avoid
-conflicts. (PT)
-
-Currently aiming for a paper of 4-8 pages that can be cited when people use or
-discuss DataCrate but this could just be treated as an abstract.
-
-This source file gets built into a [PDF file](./build/paper.pdf) - I'll update
-that periodically, but *this one* is the source file.
-
-If you want/ deserve to be an author, add your name to the [metadata
-file](./metadata.yaml).
-
-I am managing the bibliography in Zotero and exporting to bibliography.bib. If
-you can contribute by finding more references that would be great - let me know
-and I'll share the library with you or we can talk about formats.
# Note to reviewers
The specification described here is currently in draft at v0.2. A version one release is
-planned for October 2018. If accepted this paper would be updated for
+planned for October 2018. If accepted this extended abstract would be updated for
presentation and subsequent publication.
# Introduction
@@ -184,8 +164,10 @@ We think that this *laissez faire* extension mechanism in Frictionless Data
Packages is likely to result in a proliferation of highly divergent
non-standardised metadata - by using JSON-LD and specifying how to represent
temporal and geographical coverage, etc DataCrate aims to encourage common
-behaviours. In DataCrate, the approach is to use schema.org's temporalCoverage
-property. Here is an [example from v0.2](https://github.com/UTS-eResearch/datacrate/blob/22aebdcd179cb3f9b8141ca350ffafa202f5b523/spec/0.2/data_crate_specification_v0.2.md) of the specification.
+behaviours, to enable interchange of metadata. In DataCrate, the approach is to
+use schema.org's temporalCoverage property. Here is an [example from
+v0.2](https://github.com/UTS-eResearch/datacrate/blob/22aebdcd179cb3f9b8141ca350ffafa202f5b523/spec/0.2/data_crate_specification_v0.2.md)
+of the specification.
>{
> "\@id": "https://doi.org/10.5281/zenodo.1009240",
@@ -204,9 +186,13 @@ In the [DataCrate JSON-LD context] this expands to URI:
+The ability to resolve URIs is an important one if the `index.html` file is to
+be useful to humans. Many URIs in scientific ontologies do not resolve to a
+web-page but to an ontology file, which is unhelpful for people trying to
+understand what metadata terms *mean*.
-The other main alternative was the Research Object Bundle specification
-[@soiland-reyesResearchObjectBundle2014].
+ The other main alternative was the
+Research Object Bundle specification [@soiland-reyesResearchObjectBundle2014].
Rather than BagIt, the original version of Research Object Bundle uses the
Zip-based Universal Container Format - a format for which the documentation now
@@ -214,13 +200,11 @@ seems to be unavailable from Adobe and which does not have integrity features
such as checksums but there is [a version of Research Object which uses
BagIt](https://github.com/ResearchObject/bagit-ro).
-RO BagIt *does* use Linked-Data and for that reason was given
-careful consideration as a base-format for DataCrate. However, there were some
+RO BagIt *does* use Linked-Data and for that reason was given careful
+consideration as a base-format for DataCrate. However, there were some
implementation details that we thought would make it hard for tool-makers
-(including the core team at UTS).
-
-The use of "aggregations" and "annotations" introduces two extra layers of
-abstraction for describing resources.
+(including the core team at UTS); the use of "aggregations" and "annotations"
+introduces two extra layers of abstraction for describing resources.
For example, using this [sample from the bagit-ro tool]
@@ -322,15 +306,43 @@ assistance with this.
There are a number of tools for DataCrate in development.
-At the University of Technology Sydney, the Provisioner is an open framework for integrating good research data management practices into everyday research workflows. It provides a user-facing research data management planning tool which allows researchers to describe and publish datasets and create and share workspaces in different research apps such as lab notebooks, code repositories (where data is included by-reference), survey tools and collection management tools. DataCrates are used as an interchange format to move data between the different research apps, and as an ingest, archive and publication format. Lightweight adaptors coded against each research app's native API allow export and import of DataCrates, which are then used to move data from one app to another, while recording a provenance history in the DataCrates' metadata. Examples of DataCrates moving through the research lifecycle will be provided.
-
-HIEv DataCrate - At the Hawkesbury Institute for the Environment at Western Sydney University, HIEV harvests a wide range of environmental data (and associated file level metadata) from both automated sensor networks and analysed datasets generated by researchers. Leveraging built-in APIs within the HIEv a new packaging function has been developed, allowing for selected datasets to be identified and packaged in the DataCrate standard, complete with metadata automatically exported from the HIEv metadata holdings into the JSON-LD format. Going forward this will allow datasets within HIEv to be published regularly and in an automated fashion, in a format that will increase their potential for reuse.
-
-Calcytejs is a command line tool for packaging data into DataCrate developed at the University of Technology Sydney which allows researchers to describe any data set via the use of spreadsheets which the tool auto-creates in a directory tree.
-
-[Omeka DataCrate Tools](https://github.com/UTS-eResearch/omeka-datacrate-tools) is Python tool in early development to export data from Omeka Classic repositories into the DataCrate format.
-
-A tool currently in development for exporting DataCrates from the Omero microscopy repository will also be presented.
+At the University of Technology Sydney [@wheelerEndtoEndResearchData2018], the
+Provisioner is an open framework for integrating good research data management
+practices into everyday research workflows. It provides a user-facing research
+data management planning tool which allows researchers to describe and publish
+datasets and create and share workspaces in different research apps such as lab
+notebooks, code repositories (where data is included by-reference), survey tools
+and collection management tools. DataCrates are used as an interchange format to
+move data between the different research apps, and as an ingest, archive and
+publication format. Lightweight adaptors coded against each research app's
+native API allow export and import of DataCrates, which are then used to move
+data from one app to another, while recording a provenance history in the
+DataCrates' metadata. Examples of DataCrates moving through the research
+lifecycle will be provided.
+
+HIEv DataCrate - At the Hawkesbury Institute for the Environment at Western
+Sydney University, HIEV harvests a wide range of environmental data (and
+associated file level metadata) from both automated sensor networks and analysed
+datasets generated by researchers. Leveraging built-in APIs within the HIEv a
+new packaging function has been developed, allowing for selected datasets to be
+identified and packaged in the DataCrate standard, complete with metadata
+automatically exported from the HIEv metadata holdings into the JSON-LD format.
+Going forward this will allow datasets within HIEv to be published regularly and
+in an automated fashion, in a format that will increase their potential for
+reuse.
+
+[Calcytejs](https://code.research.uts.edu.au/eresearch/CalcyteJS) is a command
+line tool for packaging data into DataCrate developed at the University of
+Technology Sydney which allows researchers to describe any data set via the use
+of spreadsheets which the tool auto-creates in a directory tree.
+
+[Omeka DataCrate Tools](https://github.com/UTS-eResearch/omeka-datacrate-tools)
+is Python tool in early development to export data from Omeka Classic
+repositories into the DataCrate format.
+
+NOTE: A tool currently in development for exporting DataCrates from the Omero
+microscopy repository will also be presented, and there has been some interest
+from Caltech in adding DataCrate support to their [dataset](http://caltechlibrary.github.io/dataset) project.
# Conclusion
@@ -338,7 +350,20 @@ A tool currently in development for exporting DataCrates from the Omero microsco
DataCrate (which will be in version 1 by the time of the Research Object
workshop) has been tested on a variety of research data sets. Some examples are:
-- Data relating to the IDRC funded project (described in https://doi.org/10.3897/rio.2.e8880) [to examine data management policies and implementation for development funders](https://data.research.uts.edu.au/examples/v0.2/Data_Package-IDRC_Opportunities_and_Challenges_Open_Research_Strategies/). The project involved two parts: a review based on desk work and expert interviews and seven case studies of existing IDRC-funded projects. The case studies were supported by an Introductory Workshop in which the idea of data was examined and the issues involved in sharing discussed in detail. This was followed by an implementation phase in which the projects were supported in developing Data Management Plans. The performance against those plans was then assessed both by the participants and as part of the overall project to generate case studies that are to be published as part of the related RIO Journal Collection. The final project report will also be part of the same collection.
+- Data relating to the IDRC funded project (described in
+ https://doi.org/10.3897/rio.2.e8880) [to examine data management policies and
+ implementation for development
+ funders](https://data.research.uts.edu.au/examples/v0.2/Data_Package-IDRC_Opportunities_and_Challenges_Open_Research_Strategies/).
+ The project involved two parts: a review based on desk work and expert
+ interviews and seven case studies of existing IDRC-funded projects. The case
+ studies were supported by an Introductory Workshop in which the idea of data
+ was examined and the issues involved in sharing discussed in detail. This was
+ followed by an implementation phase in which the projects were supported in
+ developing Data Management Plans. The performance against those plans was
+ then assessed both by the participants and as part of the overall project to
+ generate case studies that are to be published as part of the related RIO
+ Journal Collection. The final project report will also be part of the same
+ collection.
- Some [Matlab code](https://data.research.uts.edu.au/examples/v0.2/GTM/) that
supports a research article. This is a good illustration of how common names
@@ -357,19 +382,28 @@ workshop) has been tested on a variety of research data sets. Some examples are:
data](https://data.research.uts.edu.au/examples/v0.2/timluckett/) - this
dataset shows how researcher affiliations can be modelled using linked data.
-Points to note (TODO):
-Looking good so far - tool developers are coming on board (Western Sydney, MIF,
-Caltech, Curtin) have all expressed interest and/or are working on tooling.
-(NOTE: This section will need to be updated at conference-time)
+At the time of the workshop, we will be in a position demonstrate a number of
+tools using DataCrate including a live data-publishing workflow implemented at
+UTS.
+
+Future plans for DataCrate include:
+- More domain specific metadata including scientific properties such as
+ temperature
+- More use-cases including preservation
+- Describing the contents of files, such as column headers in tabular data
+ formats.
+
+Some issues that would be useful to discuss at the workshop are:
+- Difficulties in finding good JSON-LD-ready ontologies for research domains.
+ (Should we be working on a schema.org extension for research?)
+- A lack of simple to use JSON-LD tools for programers (at least in Javascript
+ and Python), we have not been able to find simple to use tools to load a
+ JSON-LD file and traverse it in code.
+
-Tool support for consuming JSON-LD is still limited in some
-respects.
-Future possibilities:
-More domain specific stuff / scientific stuff such as temperature
-More use-cases including preservation
diff --git a/template.html b/template.html
new file mode 100644
index 0000000000000000000000000000000000000000..9cd5049194bfa77bad95d1dfeda2aa3c628f18c4
--- /dev/null
+++ b/template.html
@@ -0,0 +1,25 @@
+
+<html>
+<head>
+ <link rel="stylesheet"
+ href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"
+ integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u"
+ crossorigin="anonymous"/>
+
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+
+ <title>$title$</title>
+</head>
+<body>
+<h1>$title$</h1>
+$if(author)$
+ <ul>
+ $for(author)$
+ <li>$author.name$ $author.affiliation$ $author.location$ $author.email$</li>
+
+ $endfor$
+ </ul>
+$endif$
+
+ $body$
+</body>