Skip to content

Commit

Permalink
Make MDB-CEDAR synchronization idempotent
Browse files Browse the repository at this point in the history 
/api/admin/arp/initialSetup has been deprecated and replaced with
/api/admin/arp/syncMdbsWithCedar

Admin.arpSyncMdbsWithCedar() creates CEDAR templates from MDB-s, uploads
them to CEDAR and syncs them back to Dataverse ensuring that both are in
sync. syncMdbsWithCedar is idempotent, which mean it can be called any number
of times and it will generate the same results, ie. templates and elements
with always the same identifiers.

Note: there are some values in CEDAR templates, which are not represented
in MDB-s, like display values for compound types. If you have such settings
in a CEDAR template and call syncMdbsWithCedar, these will be overriden
(removed).

Also made some adjustment to convertMdbToTsv, exportMdbToCedar, and
exportTsvToCedar so that they expect an MDB name instead of an id, which
makes it more straightforward to use in any installation where the MDB names
are ficed, while ID-s may differ.
  • Loading branch information
@beepsoft
beepsoft committed Dec 11, 2023
1 parent 60af4bc commit 9c8aa01
Show file tree
Hide file tree
Showing 8 changed files with 332 additions and 150 deletions.
112 changes: 79 additions & 33 deletions src/main/java/edu/harvard/iq/dataverse/api/Admin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.json.JsonArray;
import javax.persistence.Query;
Expand Down Expand Up @@ -2339,48 +2340,93 @@ public Response arpCetCedarKey(SetCedarKeyParams params)
@EJB
ArpServiceBean arpService;

@POST
@Path("arp/initialSetup")
@Consumes("application/json")
public Response initialSetup(ArpInitialSetupParams params)
{
return Response.status(Status.NOT_FOUND).entity("/api/admin/arp/initialSetup has been deprecated, use "+
"/api/admin/arp/syncMdbsWithCedar instead").build();
}

/**
* Should be called only once when ARP is first installed.
* - Sets default namespaces for MDB-s specified in mdbNamespaceUris
* - Syncs MDB-s with CEDAR listed in syncCedar
* Synchronizes MDB-s with CEDAR templates. It uploads the selected MDB-s as CEDAR templates then syncs back
* those templates as MDB-s storing the templates and element JSON schemas along with the MDB and field types.
* This function is idempotent and can be called any number of time to do the resync time.
* @param params
* @return
*/
// curl -X POST \
// http://localhost:8080/api/admin/arp/initialSetup \
// -H 'Content-Type: application/json' \
// -d '{
// "mdbNamespaceUris": {
// "geospatial": "https://dataverse.org/schema/geospatial/",
// "socialscience": "https://dataverse.org/schema/socialscience/",
// "biomedical": "https://dataverse.org/schema/biomedical/",
// "astrophysics": "https://dataverse.org/schema/astrophysics/",
// "journal": "https://dataverse.org/schema/journal/"
// },
// "syncCedar": {
// "mdbs": [
// "citation", "journal", "geospatial", "socialscience", "astrophysics", "biomedical"
// ],
// "cedarParams": {
// "cedarDomain": "arp.orgx",
// "apiKey": "xxx",
// "folderId": "https:%2F%2Frepo.arp.orgx%2Ffolders%2F9559c51c-33e3-4429-890d-e4fa8a7de859"
// }
// }
// }'
//curl -X POST 'http://localhost:8080/api/admin/arp/syncMdbsWithCedar' \
//-H 'Content-Type: application/json' \
//-d '{
// "mdbParams": [
// {"name": "citation"},
// {"name": "geospatial", "namespaceUri": "https://dataverse.org/schema/geospatial/"},
// {"name": "socialscience", "namespaceUri": "https://dataverse.org/schema/socialscience/"},
// {"name": "biomedical", "namespaceUri": "https://dataverse.org/schema/biomedical/"},
// {"name": "astrophysics", "namespaceUri": "https://dataverse.org/schema/astrophysics/"},
// {
// "name": "journal",
// "namespaceUri": "https://dataverse.org/schema/journal/",
// "cedarUuid": "aaaaaaaa-bbbb-cccc-dddd-65d43571f306"
// }
// ],
// "cedarParams": {
// "cedarDomain": "arp3.orgx",
// "apiKey": "0000111122223333444455556666777788889999aaaabbbbccccddddeeeeffff",
// "folderId": "https:%2F%2Frepo.arp3.orgx%2Ffolders%2Fb62e9090-c9f0-4883-9bf9-010f3ae96075"
// }
//}'
@POST
@Path("arp/initialSetup")
@Path("arp/syncMdbsWithCedar")
@Consumes("application/json")
public Response arpInitialSetup(ArpInitialSetupParams params) {
public Response arpSyncMdbsWithCedar(ArpInitialSetupParams params) {
try {
if (params.getMdbNamespaceUris() != null) {
arpService.updateMetadatablockNamesaceUris(params.getMdbNamespaceUris());
if (params.getMdbParams() == null || params.getMdbParams().isEmpty()) {
return Response.serverError().entity("No mdbParams specified").build();
}
if (params.syncCedar != null && params.syncCedar.mdbs != null) {
params.syncCedar.mdbs.forEach(mdbIdtf -> {
arpService.syncMetadataBlockWithCedar(mdbIdtf, params.syncCedar.cedarParams);
});

// Make sure that both name and id are set.
try {
params.getMdbParams().stream().forEach(mdbParam -> {

if (mdbParam.id != null) {
try {
var mdb = metadataBlockSvc.findById(mdbParam.id);
mdbParam.name = mdb.getName();
} catch (IllegalArgumentException ex) {
throw new RuntimeException("Invalid MDB id " + mdbParam.id);
}
}
if (mdbParam.name != null) {
try {
var mdb = metadataBlockSvc.findByName(mdbParam.name);
mdbParam.id = mdb.getId();
} catch (IllegalArgumentException ex) {
throw new RuntimeException("Invalid MDB name " + mdbParam.name);
}
}

}
);
} catch(Exception ex) {
Logger.getLogger(Admin.class.getName()).log(Level.SEVERE, null, ex);
return Response.status(Status.BAD_REQUEST).entity(ex.getMessage()).build();
}

var namespaceUris = params.getMdbParams().stream()
.filter(mdbParam -> mdbParam.namespaceUri != null)
.collect(Collectors.toMap(mdbParam -> mdbParam.name, mdbParam -> mdbParam.namespaceUri));

if (!namespaceUris.isEmpty()) {
arpService.updateMetadatablockNamesaceUris(namespaceUris);
}

params.getMdbParams().stream().forEach(mdbParam -> {
logger.info("Syncing MDB '"+mdbParam.name+"' ...");
arpService.syncMetadataBlockWithCedar(mdbParam, params.cedarParams);
logger.info("Syncing MDB '"+mdbParam.name+"' done.");
});
return Response.ok("Done").build();
} catch (Throwable ex) {
return Response.serverError().entity(ex.getLocalizedMessage()).build();
Expand Down
60 changes: 33 additions & 27 deletions src/main/java/edu/harvard/iq/dataverse/api/arp/ArpApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.impl.CreateDatasetVersionCommand;
import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand;
import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
import edu.harvard.iq.dataverse.search.IndexServiceBean;
Expand All @@ -25,7 +24,6 @@

import javax.ejb.EJB;
import javax.inject.Inject;
import javax.json.Json;
import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
Expand Down Expand Up @@ -183,20 +181,20 @@ public Response cedarToMdb(
*
* Requires no authentication.
*
* @param mdbIdtf
* @param mdbName
* @return
*/
@GET
@Path("/convertMdbToTsv/{identifier}")
@Path("/convertMdbToTsv/{mdbName}")
@Produces("text/tab-separated-values")
public Response convertMdbToTsv(
@PathParam("identifier") String mdbIdtf
@PathParam("mdbName") String mdbName
)
{
String mdbTsv;

try {
mdbTsv = arpService.exportMdbAsTsv(mdbIdtf);
mdbTsv = arpService.exportMdbAsTsv(mdbName);
} catch (JsonProcessingException e) {
return Response.serverError().entity(e.getMessage()).build();
}
Expand All @@ -209,15 +207,15 @@ public Response convertMdbToTsv(
*
* Requires superuser authentication
*
* @param mdbIdtf
* @param mdbName
* @param cedarParams
* @return
*/
@POST
@Path("/exportMdbToCedar/{mdbIdtf}")
@Path("/exportMdbToCedar/{mdbName}")
@Consumes("application/json")
@Produces("application/json")
public Response exportMdbToCedar(@PathParam("mdbIdtf") String mdbIdtf, ExportToCedarParams cedarParams)
public Response exportMdbToCedar(@PathParam("mdbName") String mdbName, @QueryParam("uuid") String cedarUuid, ExportToCedarParams cedarParams)
{
try {
AuthenticatedUser user = findAuthenticatedUserOrDie();
Expand All @@ -239,8 +237,9 @@ public Response exportMdbToCedar(@PathParam("mdbIdtf") String mdbIdtf, ExportToC
}
cedarParams.cedarDomain = cedarDomain;

JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbIdtf)).toString());
res = arpService.exportTemplateToCedar(cedarTemplate, cedarParams);
var actualUuid = cedarUuid != null ? cedarUuid : ArpServiceBean.generateNamedUuid(mdbName);
JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName)).toString());
res = arpService.exportTemplateToCedar(cedarTemplate, actualUuid, cedarParams);
} catch (WrappedResponse ex) {
ex.printStackTrace();
return error(FORBIDDEN, "Authorized users only.");
Expand Down Expand Up @@ -282,14 +281,14 @@ public Response convertTsvToCedarTemplate(String mdbTsv)
* Exports a MetadataBlock given as TSV to CEDAR.
*
* cedarData
* @param cedarDataAndTsv
* @param data
* @return
*/
@POST
@Path("/exportTsvToCedar/")
@Consumes("application/json")
@Produces("application/json")
public Response exportTsvToCedar(ExportTsvToCedarData cedarDataAndTsv)
public Response exportTsvToCedar(ExportTsvToCedarData data)
{
try {
AuthenticatedUser user = findAuthenticatedUserOrDie();
Expand All @@ -303,8 +302,8 @@ public Response exportTsvToCedar(ExportTsvToCedarData cedarDataAndTsv)

try {
ObjectMapper mapper = new ObjectMapper();
ExportToCedarParams cedarParams = cedarDataAndTsv.cedarParams;
String cedarTsv = cedarDataAndTsv.cedarTsv;
ExportToCedarParams cedarParams = data.cedarParams;
String cedarTsv = data.tsv;

String cedarDomain = cedarParams.cedarDomain;

Expand All @@ -313,9 +312,16 @@ public Response exportTsvToCedar(ExportTsvToCedarData cedarDataAndTsv)
}
cedarParams.cedarDomain = cedarDomain;


JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(cedarTsv).toString());
arpService.exportTemplateToCedar(cedarTemplate, cedarParams);

// Use the explicitly provided UUID or create one based on the name in the TSV, ie. "schema:identifier"
// ine the CEDAR template.
var actualUuid = data.cedarUuid;
if (actualUuid == null || actualUuid.isBlank()) {
actualUuid = ArpServiceBean.generateNamedUuid(cedarTemplate.get("schema:identifier").textValue());
}

arpService.exportTemplateToCedar(cedarTemplate, actualUuid, cedarParams);
} catch (WrappedResponse ex) {
ex.printStackTrace();
return error(FORBIDDEN, "Authorized users only.");
Expand Down Expand Up @@ -364,20 +370,20 @@ public Response convertCedarTemplateToDescriboProfile(
*
* Requires no authentication.
*
* @param mdbIdtf
* @param mdbName
* @return
*/
@GET
@Path("/convertMdbToDescriboProfile/{mdbIdtf}")
@Path("/convertMdbToDescriboProfile/{mdbName}")
@Produces("application/json")
public Response convertMdbToDescriboProfile(
@PathParam("mdbIdtf") String mdbIdtf,
@PathParam("mdbName") String mdbName,
@QueryParam("lang") String language
) {
String describoProfile;

try {
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbIdtf)).toString();
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName)).toString();
describoProfile = arpService.convertTemplateToDescriboProfile(templateJson, language);
} catch (Exception e) {
e.printStackTrace();
Expand All @@ -397,25 +403,25 @@ public Response convertMdbToDescriboProfile(
* @return
*/
@GET
@Path("/convertMdbsToDescriboProfile/{identifiers}")
@Path("/convertMdbsToDescriboProfile/{mdbNames}")
@Produces("application/json")
public Response convertMdbsToDescriboProfile(
@PathParam("identifiers") String identifiers,
@PathParam("mdbNames") String identifiers,
@QueryParam("lang") String language
) {
try {
// ids separated by commas
var ids = identifiers.split(",\\s*");
// names separated by commas
var names = identifiers.split(",\\s*");
JsonObject mergedProfile = null;
JsonArray mergedProfileInputs = null;
JsonObject mergedProfileClasses = null;
JsonArray enabledClasses = null;
JsonObject layouts = null;
Gson gson = new GsonBuilder().setPrettyPrinting().create();

for (int i=0; i<ids.length; i++) {
for (int i=0; i<names.length; i++) {
// Convert TSV to CEDAR template without converting '.' to ':' in field names
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(ids[i]), false).toString();
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(names[i]), false).toString();
String profile = arpService.convertTemplateToDescriboProfile(templateJson, language);
JsonObject profileJson = gson.fromJson(profile, JsonObject.class);
boolean profileJsonAdded = false;
Expand Down
41 changes: 29 additions & 12 deletions src/main/java/edu/harvard/iq/dataverse/api/arp/ArpInitialSetupParams.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,49 @@
import edu.harvard.iq.dataverse.arp.ExportToCedarParams;

import java.util.List;
import java.util.Map;

public class ArpInitialSetupParams
{
// Default namespace URI-s for MDB-s. If a namespace URI is already set it will be updated with this one
// Key is the MDB name, value is the URI
public Map<String, String> mdbNamespaceUris;
public static class MdbParam
{
// Name of the MDB to sync. Either name or id must be provided
public String name;

// ID of the MDB to sync. Either name or id must be provided.
public Long id;

public SyncCedarData syncCedar;
// An optional namespace URI to set for the MDB in Dataverse
public String namespaceUri;

public static class SyncCedarData {
public ExportToCedarParams cedarParams;
public List<String> mdbs;
// Explicit CEDAR UUID to use when creating the template. If not provided one is automatically generated based
// on the name of the MDB.
public String cedarUuid;
}

public List<MdbParam> mdbParams;
public ExportToCedarParams cedarParams;

public ArpInitialSetupParams()
{
}

public Map<String, String> getMdbNamespaceUris()
public List<MdbParam> getMdbParams()
{
return mdbParams;
}

public void setMdbParams(List<MdbParam> mdbParams)
{
this.mdbParams = mdbParams;
}

public ExportToCedarParams getCedarParams()
{
return mdbNamespaceUris;
return cedarParams;
}

public void setMdbNamespaceUris(Map<String, String> mdbNamespaceUris)
public void setCedarParams(ExportToCedarParams cedarParams)
{
this.mdbNamespaceUris = mdbNamespaceUris;
this.cedarParams = cedarParams;
}
}
Loading

0 comments on commit 9c8aa01

Please sign in to comment.