Iqss/6497 semantic api

conf/solr/8.8.1/schema_dv_mdb_fields.xml

@@ -82,6 +82,7 @@
     <field name="keywordVocabularyURI" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="kindOfData" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="language" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="metadataOnOrig" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="northLongitude" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="notesText" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="originOfSources" type="text_en" multiValued="false" stored="true" indexed="true"/>

doc/release-notes/6497-semantic-api.md

@@ -0,0 +1,7 @@
+# Release Highlights
+
+### Dataset Semantic API (Experimental)
+
+Dataset metadata can be retrieved/set/updated using a new, flatter JSON-LD format - following the format of an OAI-ORE export (RDA-conformant Bags), allowing for easier transfer of metadata to/from other systems (i.e. without needing to know Dataverse's metadata block and field storage architecture). This new API also allows for the update of terms metadata (#5899).
+
+This development was supported by the [Research Data Alliance](https://rd-alliance.org), DANS, and Sciences PO and follows the recommendations from the [Research Data Repository Interoperability Working Group](http://dx.doi.org/10.15497/RDA00025).

doc/sphinx-guides/source/_static/api/dataset-create.jsonld

@@ -0,0 +1,15 @@
+{
+  "http://purl.org/dc/terms/title": "Darwin's Finches",
+  "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences",
+  "http://purl.org/dc/terms/creator": {
+      "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona",
+      "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc."
+  },
+  "https://dataverse.org/schema/citation/Contact": {
+    "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com",
+    "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona"
+  },
+  "https://dataverse.org/schema/citation/Description": {
+    "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds."
+  }
+}

doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst

@@ -0,0 +1,103 @@
+Dataset Semantic Metadata API
+=============================
+
+The OAI_ORE metadata export format represents Dataset metadata using json-ld (see the :doc:`/admin/metadataexport` section). As part of an RDA-supported effort to allow import of Datasets exported as Bags with an included OAI_ORE metadata file, 
+an experimental API has been created that provides a json-ld alternative to the v1.0 API calls to get/set/delete Dataset metadata in the :doc:`/api/native-api`.
+
+You may prefer to work with this API if you are building a tool to import from a Bag/OAI-ORE source or already work with json-ld representations of metadata, or if you prefer the flatter json-ld representation to Dataverse software's json representation (which includes structure related to the metadata blocks involved and the type/multiplicity of the metadata fields.) 
+You may not want to use this API if you need stability and backward compatibility (the 'experimental' designation for this API implies that community feedback is desired and that, in future Dataverse software versions, the API may be modified based on that feedback).
+
+Note: The examples use the 'application/ld+json' mimetype. For compatibility reasons, the APIs also be used with mimetype "application/json-ld"
+
+Get Dataset Metadata
+--------------------
+
+To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID), and, for specific versions, the version number.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export DATASET_ID='12345'
+  export DATASET_PID='doi:10.5072/FK2A1B2C3'
+  export VERSION='1.0'
+  export SERVER_URL=https://demo.dataverse.org
+
+  Example 1: Get metadata for version '1.0'
+
+    curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/$DATASET_ID/versions/$VERSION/metadata"
+
+  Example 2: Get metadata for the latest version using the DATASET PID
+
+    curl -H X-Dataverse-key:$API_TOKEN -H 'Accept: application/ld+json' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID"
+
+You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object.
+
+
+Add Dataset Metadata
+--------------------
+
+To add json-ld formatted metadata for a Dataset, specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID). Adding '?replace=true' will overwrite an existing metadata value. The default (replace=false) will only add new metadata or add a new value to a multi-valued field. 
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export DATASET_ID='12345'
+  export DATASET_PID='doi:10.5072/FK2A1B2C3'
+  export VERSION='1.0'
+  export SERVER_URL=https://demo.dataverse.org
+
+  Example: Change the Dataset title 
+
+    curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"Title": "Submit menu test", "@context":{"Title": "http://purl.org/dc/terms/title"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?replace=true"
+
+  Example 2: Add a description using the DATASET PID
+
+    curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"citation:Description": {"dsDescription:Text": "New description"}, "@context":{"citation": "https://dataverse.org/schema/citation/","dsDescription": "https://dataverse.org/schema/citation/dsDescription#"}}' "$SERVER_URL/api/datasets/:persistentId/metadata?persistentId=$DATASET_PID"
+
+You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated.
+
+
+Delete Dataset Metadata
+-----------------------
+
+To delete metadata for a Dataset, send a json-ld representation of the fields to delete and specify the Dataset ID (DATASET_ID) or Persistent identifier (DATASET_PID).
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export DATASET_ID='12345'
+  export DATASET_PID='doi:10.5072/FK2A1B2C3'
+  export VERSION='1.0'
+  export SERVER_URL=https://demo.dataverse.org
+
+  Example: Delete the TermsOfUseAndAccess 'restrictions' value 'No restrictions' for the latest version using the DATASET PID
+
+    curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"https://dataverse.org/schema/core#restrictions":"No restrictions"}' "$SERVER_URL/api/datasets/:persistentId/metadata/delete?persistentId=$DATASET_PID"
+
+Note, this example uses the term URI directly rather than adding an '@context' element. You can use either form in any of these API calls. 
+
+You should expect a 200 ("OK") response indicating whether a draft Dataset version was created or an existing draft was updated.
+
+
+Create a Dataset
+----------------
+
+Specifying the Content-Type as application/ld+json with the existing /api/dataverses/{id}/datasets API call (see :ref:`create-dataset-command`) supports using the same metadata format when creating a Dataset.
+
+With curl, this is done by adding the following header:
+
+.. code-block:: bash
+
+  -H 'Content-Type: application/ld+json' 
+
+  .. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export DATAVERSE_ID=root
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+
+  curl -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets --upload-file dataset-create.jsonld
+
+An example jsonld file is available at :download:`dataset-create.jsonld <../_static/api/dataset-create.jsonld>` 
+

doc/sphinx-guides/source/developers/index.rst

@@ -35,4 +35,5 @@ Developer Guide
    big-data-support
    aux-file-support
    s3-direct-upload-api
+   dataset-semantic-metadata-api
    workflows

pom.xml

@@ -173,9 +173,21 @@
         <dependency>
             <groupId>org.glassfish</groupId>
             <artifactId>javax.json</artifactId>
-            <version>1.0.4</version>
+            <version>1.1.4</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.skyscreamer</groupId>
+            <artifactId>jsonassert</artifactId>
+            <version>1.5.0</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.vaadin.external.google</groupId>
+                    <artifactId>android-json</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
@@ -218,6 +230,11 @@
             <artifactId>aws-java-sdk-s3</artifactId>
             <!-- no version here as managed by BOM above! -->
         </dependency>
+        <dependency>
+          <groupId>com.apicatalog</groupId>
+          <artifactId>titanium-json-ld</artifactId>
+          <version>0.8.6</version>
+        </dependency>
         <dependency>
             <!-- required by org.swordapp.server.sword2-server -->
             <groupId>org.apache.abdera</groupId>

scripts/api/data/metadatablocks/migration.tsv

@@ -0,0 +1,5 @@
+#metadataBlock	name	dataverseAlias	displayName	blockURI											
+	migration		Migrated Metadata	https://dataverse.org/schema/migration/											
+#datasetField	name	title	description	watermark	 fieldType	displayOrder	displayFormat	advancedSearchField	allowControlledVocabulary	allowmultiples	facetable	displayoncreate	required	parent	metadatablock_id	termURI
+	metadataOnOrig	Metadata on the original source of migrated datasets.			textbox	1		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		migration	https://dataverse.org/schema/core#metadataOnOrig
+#controlledVocabulary	DatasetField	Value	identifier	displayOrder											

scripts/search/tests/data/dataset-finch1.jsonld

@@ -0,0 +1,26 @@
+
+{
+  "http://purl.org/dc/terms/title": "Darwin's Finches",
+  "http://purl.org/dc/terms/subject": "Medicine, Health and Life Sciences",
+  "http://purl.org/dc/terms/creator": {
+      "https://dataverse.org/schema/citation/author#Name": "Finch, Fiona",
+      "https://dataverse.org/schema/citation/author#Affiliation": "Birds Inc."
+  },
+  "https://dataverse.org/schema/citation/Contact": {
+    "https://dataverse.org/schema/citation/datasetContact#E-mail": "finch@mailinator.com",
+    "https://dataverse.org/schema/citation/datasetContact#Name": "Finch, Fiona"
+  },
+  "https://dataverse.org/schema/citation/Description": {
+    "https://dataverse.org/schema/citation/dsDescription#Text": "Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds."
+  },
+  "@type": [
+    "http://www.openarchives.org/ore/terms/Aggregation",
+    "http://schema.org/Dataset"
+  ],
+  "http://schema.org/version": "DRAFT",
+  "http://schema.org/name": "Darwin's Finches",
+    "https://dataverse.org/schema/core#fileTermsOfAccess": {
+        "https://dataverse.org/schema/core#fileRequestAccess": false
+    },
+    "http://schema.org/includedInDataCatalog": "Root"
+}

src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java

@@ -1863,7 +1863,7 @@ public String getJsonLd() {
             JsonObjectBuilder license = Json.createObjectBuilder().add("@type", "Dataset");
 
             if (TermsOfUseAndAccess.License.CC0.equals(terms.getLicense())) {
-                license.add("text", "CC0").add("url", "https://creativecommons.org/publicdomain/zero/1.0/");
+                license.add("text", "CC0").add("url", TermsOfUseAndAccess.CC0_URI);
             } else {
                 String termsOfUse = terms.getTermsOfUse();
                 // Terms of use can be null if you create the dataset with JSON.

src/main/java/edu/harvard/iq/dataverse/TermsOfUseAndAccess.java

@@ -280,7 +280,7 @@ public enum License {
      * API use? See also https://github.com/IQSS/dataverse/issues/1385
      */
     public static TermsOfUseAndAccess.License defaultLicense = TermsOfUseAndAccess.License.CC0;
-
+    public static String CC0_URI = "https://creativecommons.org/publicdomain/zero/1.0/";
     @Override
     public int hashCode() {
         int hash = 0;

src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java

@@ -137,6 +137,7 @@ public Response getByName(@PathParam("name") String name) {
             String solrFieldSearchable = dsf.getSolrField().getNameSearchable();
             String solrFieldFacetable = dsf.getSolrField().getNameFacetable();
             String metadataBlock = dsf.getMetadataBlock().getName();
+            String uri=dsf.getUri();
             boolean hasParent = dsf.isHasParent();
             boolean allowsMultiples = dsf.isAllowMultiples();
             boolean isRequired = dsf.isRequired();
@@ -168,7 +169,8 @@ public Response getByName(@PathParam("name") String name) {
                     .add("parentAllowsMultiples", parentAllowsMultiplesDisplay)
                     .add("solrFieldSearchable", solrFieldSearchable)
                     .add("solrFieldFacetable", solrFieldFacetable)
-                    .add("isRequired", isRequired));
+                    .add("isRequired", isRequired)
+                    .add("uri", uri));
 
         } catch ( NoResultException nre ) {
             return notFound(name);
@@ -356,7 +358,7 @@ public String getArrayIndexOutOfBoundMessage(HeaderType header,
                                                  int wrongIndex) {
 
         List<String> columns = getColumnsByHeader(header);
-
+        
         String column = columns.get(wrongIndex - 1);
         List<String> arguments = new ArrayList<>();
         arguments.add(header.name());