From 695709c200c3fa01f5c291486f8d8ad296ca52b0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:51:52 -0400 Subject: [PATCH 1/7] standardize image url --- .../iq/dataverse/DataverseServiceBean.java | 10 ++++- .../iq/dataverse/ThumbnailServiceWrapper.java | 20 ++++++++- .../dataverse/search/SearchServiceBean.java | 4 +- .../harvard/iq/dataverse/api/SearchIT.java | 43 +++++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 31fd775ffdf..65082011eba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -361,7 +361,15 @@ public String getDataverseLogoThumbnailAsBase64ById(Long dvId) { } return null; } - + + public String getDataverseLogoThumbnailAsUrl(Long dvId) { + File dataverseLogoFile = getLogoById(dvId); + if (dataverseLogoFile != null) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dvCardImage/" + dvId; + } + return null; + } + private File getLogo(Dataverse dataverse) { if (dataverse.getId() == null) { return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index c1127079da4..91d25ba2d38 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -48,6 +48,18 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map dvobjectViewMap = new HashMap<>(); private Map hasThumbMap = new HashMap<>(); + public String getFileCardImageAsUrl(SolrSearchResult result) { + if (result.isHarvested()) { + return null; + } + + if (result.getEntity() == null) { + return null; + } + + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + } + // it's the responsibility of the user - to make sure the search result // passed to this method is of the Datafile type! public String getFileCardImageAsBase64Url(SolrSearchResult result) { @@ -208,7 +220,13 @@ public String getDatasetCardImageAsUrl(Dataset dataset, Long versionId, boolean public String getDataverseCardImageAsBase64Url(SolrSearchResult result) { return dataverseService.getDataverseLogoThumbnailAsBase64ById(result.getEntityId()); } - + + // it's the responsibility of the user - to make sure the search result + // passed to this method is of the Dataverse type! + public String getDataverseCardImageAsUrl(SolrSearchResult result) { + return dataverseService.getDataverseLogoThumbnailAsUrl(result.getEntityId()); + } + public void resetObjectMaps() { dvobjectThumbnailsMap = new HashMap<>(); dvobjectViewMap = new HashMap<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 28676caeac5..ee93c49ad34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -593,7 +593,7 @@ public SolrQueryResponse search( solrSearchResult.setDataverseAffiliation(dataverseAffiliation); solrSearchResult.setDataverseParentAlias(dataverseParentAlias); solrSearchResult.setDataverseParentName(dataverseParentName); - solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsBase64Url(solrSearchResult)); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsUrl(solrSearchResult)); /** * @todo Expose this API URL after "dvs" is changed to * "dataverses". Also, is an API token required for published @@ -652,7 +652,7 @@ public SolrQueryResponse search( } solrSearchResult.setHtmlUrl(baseUrl + "/dataset.xhtml?persistentId=" + parentGlobalId); solrSearchResult.setDownloadUrl(baseUrl + "/api/access/datafile/" + entityid); - solrSearchResult.setImageUrl(thumbnailServiceWrapper.getFileCardImageAsBase64Url(solrSearchResult)); + solrSearchResult.setImageUrl(thumbnailServiceWrapper.getFileCardImageAsUrl(solrSearchResult)); /** * @todo We are not yet setting the API URL for files because * not all files have metadata. Only subsettable files (those diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index c743c12c7de..4b422002cf2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1283,4 +1283,47 @@ public void tearDownDataverse() { public static void cleanup() { } + @Test + public void testSearchFiles() { + Response createUser = UtilIT.createRandomUser(); + createUser.prettyPrint(); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.prettyPrint(); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + System.out.println("id: " + datasetId); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + System.out.println("datasetPid: " + datasetPid); + + String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + + Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat() + .statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response searchResp = UtilIT.search("dataverseproject", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].file_content_type", CoreMatchers.is("image/png")) + .body("data.items[0].url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("/api/access/datafile/")) + .body("data.items[0].image_url", CoreMatchers.containsString("imageThumb=true")); + } } From c80a3dffad78474a19972e5cab8e81d13e78a454 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:47:41 -0400 Subject: [PATCH 2/7] doc changes --- ...831-standardize-image-url-of-search-api.md | 28 +++++++++++++++++++ doc/sphinx-guides/source/api/changelog.rst | 5 ++++ 2 files changed, 33 insertions(+) create mode 100644 doc/release-notes/10831-standardize-image-url-of-search-api.md diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md new file mode 100644 index 00000000000..3d9216a7c09 --- /dev/null +++ b/doc/release-notes/10831-standardize-image-url-of-search-api.md @@ -0,0 +1,28 @@ +Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. +Note to release not writer: this supersedes the release note 10810-search-api-payload-extensions.md + +For Dataverse: + +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "image_url":"/api/access/dvCardImage/{identifier}" +(etc, etc) +``` + +For DataFile: + +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.txt", + ... + "image_url":"/api/access/datafile/{identifier}?imageThumb=true" +(etc, etc) +``` diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index a7af3e84b28..8d11ac0c123 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.4 +---- + +- ** /api/search?q=**: Json values for image_url in DataFiles and Collections have changed from Base64URL ("data:image/png;base64,...) to "/api/access/datafile/{identifier}?imageThumb=true" and "/api/access/dvCardImage/{identifier}" respectively. This was done to match the image_url of Dataset. + v6.3 ---- From b923814e5d1458824a357f6d0984fcbda29cef68 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 09:48:27 -0400 Subject: [PATCH 3/7] doc changes --- doc/release-notes/10831-standardize-image-url-of-search-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md index 3d9216a7c09..1910091455c 100644 --- a/doc/release-notes/10831-standardize-image-url-of-search-api.md +++ b/doc/release-notes/10831-standardize-image-url-of-search-api.md @@ -1,5 +1,5 @@ Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. -Note to release not writer: this supersedes the release note 10810-search-api-payload-extensions.md +Note to release note writer: this supersedes the release note 10810-search-api-payload-extensions.md For Dataverse: From 18c921107dc57478081132be9718a66cf123164f Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:11:59 -0400 Subject: [PATCH 4/7] doc changes --- doc/sphinx-guides/source/api/search.rst | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 359313ce1b5..7ca9a5abca6 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -61,7 +61,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/trees", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/1", "identifier":"trees", "description":"A tree dataverse with some birds", "published_at":"2016-05-10T12:53:38Z", @@ -76,7 +76,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Chestnut Trees", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/chestnuttrees", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"chestnuttrees", "description":"A dataverse with chestnut trees and an oriole", "published_at":"2016-05-10T12:52:38Z", @@ -91,7 +91,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"trees.png", "type":"file", "url":"https://demo.dataverse.org/api/access/datafile/12", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/datafile/12?imageThumb=true", "file_id":"12", "description":"", "published_at":"2016-05-10T12:53:39Z", @@ -113,7 +113,7 @@ https://demo.dataverse.org/api/search?q=trees "name":"Birds", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/birds", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/3", "identifier":"birds", "description":"A bird Dataverse collection with some trees", "published_at":"2016-05-10T12:57:27Z", @@ -173,8 +173,6 @@ https://demo.dataverse.org/api/search?q=trees } } -Note that the image_url field, if exists, will be returned as a regular URL for Datasets, while for Files and Dataverses, it will be returned as a Base64 URL. We plan to standardize this behavior so that the field always returns a regular URL. (See: https://github.com/IQSS/dataverse/issues/10831) - .. _advancedsearch-example: Advanced Search Examples @@ -202,7 +200,7 @@ In this example, ``show_relevance=true`` matches per field are shown. Available "name":"Finches", "type":"dataverse", "url":"https://demo.dataverse.org/dataverse/finches", - "image_url":"data:image/png;base64,iVBORw0...", + "image_url":"https://demo.dataverse.org/api/access/dvCardImage/2", "identifier":"finches", "description":"A Dataverse collection with finches", "published_at":"2016-05-10T12:57:38Z", From 47a2233b76ead9b57f9b96d11ae6c4f6d93dfe36 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:55:52 -0400 Subject: [PATCH 5/7] doc changes --- doc/sphinx-guides/source/api/changelog.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 8d11ac0c123..a7af3e84b28 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,11 +7,6 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 -v6.4 ----- - -- ** /api/search?q=**: Json values for image_url in DataFiles and Collections have changed from Base64URL ("data:image/png;base64,...) to "/api/access/datafile/{identifier}?imageThumb=true" and "/api/access/dvCardImage/{identifier}" respectively. This was done to match the image_url of Dataset. - v6.3 ---- From 48f9f72ac6cb8b6995c10c3375c9aa756e3532ba Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:55:24 -0400 Subject: [PATCH 6/7] dont return image_url if there is none --- .../iq/dataverse/DataverseServiceBean.java | 2 +- .../iq/dataverse/ThumbnailServiceWrapper.java | 11 ++++----- .../harvard/iq/dataverse/api/SearchIT.java | 23 ++++++++++++++++++- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 65082011eba..91b15f77111 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -364,7 +364,7 @@ public String getDataverseLogoThumbnailAsBase64ById(Long dvId) { public String getDataverseLogoThumbnailAsUrl(Long dvId) { File dataverseLogoFile = getLogoById(dvId); - if (dataverseLogoFile != null) { + if (dataverseLogoFile != null && dataverseLogoFile.exists()) { return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dvCardImage/" + dvId; } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index 91d25ba2d38..36be7c61642 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -49,15 +49,14 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map hasThumbMap = new HashMap<>(); public String getFileCardImageAsUrl(SolrSearchResult result) { - if (result.isHarvested()) { - return null; - } - if (result.getEntity() == null) { - return null; + if (!result.isHarvested() && result.getEntity() != null && (!((DataFile)result.getEntity()).isRestricted() + || permissionsWrapper.hasDownloadFilePermission(result.getEntity())) + && isThumbnailAvailable((DataFile) result.getEntity())) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; } - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + return null; } // it's the responsibility of the user - to make sure the search result diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index 4b422002cf2..3a2b684c421 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -1284,7 +1284,7 @@ public static void cleanup() { } @Test - public void testSearchFiles() { + public void testSearchFilesAndUrlImages() { Response createUser = UtilIT.createRandomUser(); createUser.prettyPrint(); String username = UtilIT.getUsernameFromResponse(createUser); @@ -1304,6 +1304,11 @@ public void testSearchFiles() { String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); uploadImage.prettyPrint(); + uploadImage.then().assertThat() + .statusCode(200); + pathToFile = "src/main/webapp/resources/js/mydata.js"; + Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); + uploadImage.prettyPrint(); uploadImage.then().assertThat() .statusCode(200); @@ -1325,5 +1330,21 @@ public void testSearchFiles() { .body("data.items[0].url", CoreMatchers.containsString("/api/access/datafile/")) .body("data.items[0].image_url", CoreMatchers.containsString("/api/access/datafile/")) .body("data.items[0].image_url", CoreMatchers.containsString("imageThumb=true")); + + searchResp = UtilIT.search(dataverseAlias, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("dataverse")) + .body("data.items[0].url", CoreMatchers.containsString("/dataverse/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image"))); + + searchResp = UtilIT.search("mydata", apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("url_image"))); } } From d215221b8b884b55a1ef2c1468deb2e4c799a9b4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:20:51 -0400 Subject: [PATCH 7/7] adding fix from review comment --- .../iq/dataverse/ThumbnailServiceWrapper.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index 36be7c61642..542cf39cfbe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -10,6 +10,7 @@ import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.search.SolrSearchResult; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; @@ -49,14 +50,16 @@ public class ThumbnailServiceWrapper implements java.io.Serializable { private Map hasThumbMap = new HashMap<>(); public String getFileCardImageAsUrl(SolrSearchResult result) { - - if (!result.isHarvested() && result.getEntity() != null && (!((DataFile)result.getEntity()).isRestricted() - || permissionsWrapper.hasDownloadFilePermission(result.getEntity())) - && isThumbnailAvailable((DataFile) result.getEntity())) { - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + result.getEntity().getId() + "?imageThumb=true"; + DataFile dataFile = result != null && result.getEntity() != null ? ((DataFile) result.getEntity()) : null; + if (dataFile == null || result.isHarvested() + || !isThumbnailAvailable(dataFile) + || dataFile.isRestricted() + || !dataFile.isReleased() + || FileUtil.isActivelyEmbargoed(dataFile) + || FileUtil.isRetentionExpired(dataFile)) { + return null; } - - return null; + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/datafile/" + dataFile.getId() + "?imageThumb=true"; } // it's the responsibility of the user - to make sure the search result