Skip to content

Commit

Permalink
Refact TSV to CEDAR template generation using optional existing template
Browse files Browse the repository at this point in the history 
When converting a metadatablock, ie. a TSV  to a CEDAR template we can have
two cases

1. If it is an initial generation, we generate the CEDAR template solely based
on the data stored in the DB tables

2. If we have already generated a CEDAR template for the MDB and exported it
to CEDAR, then we will also load it back right away and store it in
MetadatablockArp as well as for compund fields in DatasetFielTypeArp. In this
case when we generate a CEDAR template based on a TSV/MDB we load the
previously known CEDAR template representation and augment it with date
coming from the TSV. This allows updating a CEDAR template with changed
MDB values while keeping values in the CEDAR template that are not
related to MDB-s, ie. some "_arp" values that are only used by AROMA via
profiles.
  • Loading branch information
@beepsoft
beepsoft committed Dec 12, 2023
1 parent 9c8aa01 commit 39d1947
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 48 deletions.
13 changes: 8 additions & 5 deletions src/main/java/edu/harvard/iq/dataverse/api/arp/ArpApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,9 @@ public Response exportMdbToCedar(@PathParam("mdbName") String mdbName, @QueryPar
}
cedarParams.cedarDomain = cedarDomain;

JsonObject existingTemplate = arpService.getCedarTemplateForMdb(mdbName);
var actualUuid = cedarUuid != null ? cedarUuid : ArpServiceBean.generateNamedUuid(mdbName);
JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName)).toString());
JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName), existingTemplate).toString());
res = arpService.exportTemplateToCedar(cedarTemplate, actualUuid, cedarParams);
} catch (WrappedResponse ex) {
ex.printStackTrace();
Expand Down Expand Up @@ -268,7 +269,7 @@ public Response convertTsvToCedarTemplate(String mdbTsv)
String cedarTemplate;

try {
cedarTemplate = arpService.tsvToCedarTemplate(mdbTsv).toString();
cedarTemplate = arpService.tsvToCedarTemplate(mdbTsv, null).toString();
} catch (JsonProcessingException e) {
e.printStackTrace();
return Response.serverError().entity(e.getMessage()).build();
Expand Down Expand Up @@ -312,7 +313,7 @@ public Response exportTsvToCedar(ExportTsvToCedarData data)
}
cedarParams.cedarDomain = cedarDomain;

JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(cedarTsv).toString());
JsonNode cedarTemplate = mapper.readTree(arpService.tsvToCedarTemplate(cedarTsv, null).toString());

// Use the explicitly provided UUID or create one based on the name in the TSV, ie. "schema:identifier"
// ine the CEDAR template.
Expand Down Expand Up @@ -383,7 +384,8 @@ public Response convertMdbToDescriboProfile(
String describoProfile;

try {
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName)).toString();
JsonObject existingTemplate = arpService.getCedarTemplateForMdb(mdbName);
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(mdbName), existingTemplate).toString();
describoProfile = arpService.convertTemplateToDescriboProfile(templateJson, language);
} catch (Exception e) {
e.printStackTrace();
Expand Down Expand Up @@ -421,7 +423,8 @@ public Response convertMdbsToDescriboProfile(

for (int i=0; i<names.length; i++) {
// Convert TSV to CEDAR template without converting '.' to ':' in field names
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(names[i]), false).toString();
JsonObject existingTemplate = arpService.getCedarTemplateForMdb(names[i]);
String templateJson = arpService.tsvToCedarTemplate(arpService.exportMdbAsTsv(names[i]), false, existingTemplate).toString();
String profile = arpService.convertTemplateToDescriboProfile(templateJson, language);
JsonObject profileJson = gson.fromJson(profile, JsonObject.class);
boolean profileJsonAdded = false;
Expand Down
56 changes: 47 additions & 9 deletions src/main/java/edu/harvard/iq/dataverse/arp/ArpServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,25 @@ public static JsonObject loadJsonFromResource(String resourcePath) {
}
}

/**
* Get the CEDAR template json for the given MDB from the associated MetadatablockArp record, ie. the
* CEDAR representation of the MDB as we've last seen it during an export from CEDAR to DV.
* @param mdbName
* @return
*/
public JsonObject getCedarTemplateForMdb(String mdbName) {
var mdb = metadataBlockService.findByName(mdbName);
if (mdb == null) {
return null;
}
var mdbArp = arpMetadataBlockServiceBean.findMetadataBlockArpForMetadataBlock(mdb);
if (mdbArp == null) {
return null;
}
JsonObject cedarTemplate = new Gson().fromJson(mdbArp.getCedarDefinition(), JsonObject.class);
return cedarTemplate;
}

public String exportMdbAsTsv(String mdbName) throws JsonProcessingException {
MetadataBlock mdb = metadataBlockService.findByName(mdbName);

Expand Down Expand Up @@ -229,13 +248,27 @@ public String exportMdbAsTsv(String mdbName) throws JsonProcessingException {
return metadataBlocks + "\n" + datasetFieldValues + "\n" + controlledVocabularyValues;
}

public JsonObject tsvToCedarTemplate(String tsv) throws JsonProcessingException {
return tsvToCedarTemplate(tsv, true);
public JsonObject tsvToCedarTemplate(String tsv, JsonObject existingTemplate) throws JsonProcessingException {
return tsvToCedarTemplate(tsv, true, existingTemplate);
}

public JsonObject tsvToCedarTemplate(String tsv, boolean convertDotToColon) throws JsonProcessingException {
var converter = new TsvToCedarTemplate(convertDotToColon);
return converter.convert(tsv);
/**
* Convert the given tsv to a CEDAR template optionally using existingTemplate as the base of the
* generated final CEDAR template. By providing an existingTemplate we can keep CEDAR specific values
* that are not represented in an MDB while also being able to set MDB specific value on the
* CEDAR template. For example, if a name is updated in MDB and we want to sync it back to CEDAR
* we can take the MDB's CEDAR template representation in MetadatablockArp as the existingTemplate
* and apply what we currently have in MDB (ie. including an updated name).
*
* @param tsv
* @param convertDotToColon
* @param existingTemplate
* @return
* @throws JsonProcessingException
*/
public JsonObject tsvToCedarTemplate(String tsv, boolean convertDotToColon, JsonObject existingTemplate) throws JsonProcessingException {
var converter = new TsvToCedarTemplate(tsv, convertDotToColon, existingTemplate);
return converter.convert();
}

private static boolean isURLEncoded(String param) {
Expand Down Expand Up @@ -429,7 +462,8 @@ private List<JsonNode> exportElements(JsonNode cedarTemplate, ExportToCedarParam
prop = prop.get("items");
cedarUuid = prop.get("@id").textValue();
}
String cedarId = "https://repo." + cedarDomain + "/template-elements/" + cedarUuid;
// If cedarUuid is already an URL use it as-is, otherwise generate one based in @id being a uuid
String cedarId = cedarUuid.startsWith("http") ? cedarUuid : "https://repo." + cedarDomain + "/template-elements/" + cedarUuid;
String cedarIdEncoded = URLEncoder.encode(cedarId);
String resUrl = "https://resource." + cedarDomain + "/template-elements/"+cedarIdEncoded;

Expand All @@ -445,10 +479,11 @@ private List<JsonNode> exportElements(JsonNode cedarTemplate, ExportToCedarParam

// if element exists, just update
if (elementExistsResponse.statusCode() == 200) {
String json = new ObjectMapper().writeValueAsString(prop);
HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(new URI(resUrl))
.headers("Authorization", "apiKey " + apiKey, "Content-Type", "application/json", "Accept", "application/json")
.PUT(HttpRequest.BodyPublishers.ofString(new ObjectMapper().writeValueAsString(prop)))
.PUT(HttpRequest.BodyPublishers.ofString(json))
.build();

// The right is the GET request to get the Element after update
Expand All @@ -458,10 +493,11 @@ private List<JsonNode> exportElements(JsonNode cedarTemplate, ExportToCedarParam
// Create the element
else {
String urlWithFolder = resUrl+"?folder_id=" + encodedFolderId;
String json = new ObjectMapper().writeValueAsString(prop);
HttpRequest httpRequest = HttpRequest.newBuilder()
.uri(new URI(urlWithFolder))
.headers("Authorization", "apiKey " + apiKey, "Content-Type", "application/json", "Accept", "application/json")
.PUT(HttpRequest.BodyPublishers.ofString(new ObjectMapper().writeValueAsString(prop)))
.PUT(HttpRequest.BodyPublishers.ofString(json))
.build();
// the right null marks that no need for a GET after this request
templateElementsRequests.add(new ImmutablePair<>(httpRequest, null));
Expand Down Expand Up @@ -1238,7 +1274,9 @@ public void syncMetadataBlockWithCedar(ArpInitialSetupParams.MdbParam mdbParam,

var mdb = metadataBlockService.findByName(mdbParam.name);
var actualUuid = mdbParam.cedarUuid != null ? mdbParam.cedarUuid : generateNamedUuid(mdbParam.name);
JsonNode cedarTemplate = mapper.readTree(tsvToCedarTemplate(exportMdbAsTsv(mdb.getName())).toString());

JsonObject existingTemplate = getCedarTemplateForMdb(mdbParam.name);
JsonNode cedarTemplate = mapper.readTree(tsvToCedarTemplate(exportMdbAsTsv(mdb.getName()), existingTemplate).toString());
String templateJson = exportTemplateToCedar(cedarTemplate, actualUuid, cedarParams);
createOrUpdateMdbFromCedarTemplate("root", templateJson, false);
} catch (Exception e) {
Expand Down
Loading

0 comments on commit 39d1947

Please sign in to comment.