From 9457fec308a7c5537befcdb4c150134081226383 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 3 Feb 2020 13:46:31 -0500 Subject: [PATCH] #6524 add api endpoints for file storage/download size at dataset/version level --- .../iq/dataverse/DatasetServiceBean.java | 60 ++++++++++++++----- .../harvard/iq/dataverse/api/Datasets.java | 16 ++++- .../impl/GetDatasetStorageSizeCommand.java | 26 ++++++-- .../impl/GetDataverseStorageSizeCommand.java | 2 +- .../edu/harvard/iq/dataverse/api/FilesIT.java | 16 ++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 9 ++- 6 files changed, 103 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 530c703ef97..661b3cda448 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -866,45 +866,73 @@ public void obtainPersistentIdentifiersForDatafiles(Dataset dataset) { } public long findStorageSize(Dataset dataset) throws IOException { - return findStorageSize(dataset, false); + return findStorageSize(dataset, false, "storage", null); } + + public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws IOException { + return findStorageSize(dataset, countCachedExtras, "storage", null); + } + /** * Returns the total byte size of the files in this dataset * * @param dataset * @param countCachedExtras boolean indicating if the cached disposable extras should also be counted + * @param mode String indicating whether we are getting the result for storage (entire dataset) or download version based + * @param version optional param for dataset version * @return total size * @throws IOException if it can't access the objects via StorageIO * (in practice, this can only happen when called with countCachedExtras=true; when run in the * default mode, the method doesn't need to access the storage system, as the * sizes of the main files are recorded in the database) */ - public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws IOException { + public long findStorageSize(Dataset dataset, boolean countCachedExtras, String mode, DatasetVersion version) throws IOException { long total = 0L; if (dataset.isHarvested()) { return 0L; } - for (DataFile datafile : dataset.getFiles()) { - total += datafile.getFilesize(); + System.out.print("mode: " + mode); + System.out.print("dataset: " + dataset); + System.out.print("version: " + version); + List filesToTalley = new ArrayList(); + + if (version == null || (mode != null && mode.equals("storage"))){ + filesToTalley = dataset.getFiles(); + } else { + List fmds = version.getFileMetadatas(); + System.out.print("FileMetadata: " + fmds.size()); + for (FileMetadata fmd : fmds){ + filesToTalley.add(fmd.getDataFile()); + } - if (!countCachedExtras) { - if (datafile.isTabularData()) { - // count the size of the stored original, in addition to the main tab-delimited file: - Long originalFileSize = datafile.getDataTable().getOriginalFileSize(); - if (originalFileSize != null) { - total += originalFileSize; + } + + System.out.print("filesToTalley: " + filesToTalley.size()); + + //CACHED EXTRAS FOR DOWNLOAD? + + + for (DataFile datafile : filesToTalley) { + total += datafile.getFilesize(); + + if (!countCachedExtras) { + if (datafile.isTabularData()) { + // count the size of the stored original, in addition to the main tab-delimited file: + Long originalFileSize = datafile.getDataTable().getOriginalFileSize(); + if (originalFileSize != null) { + total += originalFileSize; + } + } + } else { + StorageIO storageIO = datafile.getStorageIO(); + for (String cachedFileTag : storageIO.listAuxObjects()) { + total += storageIO.getAuxObjectSize(cachedFileTag); } } - } else { - StorageIO storageIO = datafile.getStorageIO(); - for (String cachedFileTag : storageIO.listAuxObjects()) { - total += storageIO.getAuxObjectSize(cachedFileTag); - } } - } // and finally, if (countCachedExtras) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index eb36cb472d9..5c9a0624608 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1802,10 +1802,20 @@ public Response getMakeDataCountMetricCurrentMonth(@PathParam("id") String idSup @GET @Path("{identifier}/storagesize") - public Response getStorageSize(@PathParam("identifier") String dvIdtf, @QueryParam("includeCached") boolean includeCached) throws WrappedResponse { - + public Response getStorageSize(@PathParam("identifier") String dvIdtf, @QueryParam("includeCached") boolean includeCached, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize"), + execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached, "storage", null))))); + } + + @GET + @Path("{identifier}/versions/{versionId}/downloadsize") + public Response getDownloadSize(@PathParam("identifier") String dvIdtf, @PathParam("versionId") String version, + @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize"), - execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached))))); + execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, "download", getDatasetVersionOrDie(req, version , findDatasetOrDie(dvIdtf), uriInfo, headers)))))); } @GET diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java index 5fc1cef9312..df1897ee243 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java @@ -6,6 +6,9 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -13,6 +16,9 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Set; import java.util.logging.Logger; /** @@ -26,36 +32,48 @@ public class GetDatasetStorageSizeCommand extends AbstractCommand { private final Dataset dataset; private final Boolean countCachedFiles; + private final String mode; + private final DatasetVersion version; public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target) { super(aRequest, target); dataset = target; countCachedFiles = false; + mode = "download"; + version = null; } - public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles) { + public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, String mode, DatasetVersion version) { super(aRequest, target); dataset = target; this.countCachedFiles = countCachedFiles; + this.mode = mode; + this.version = version; } @Override public Long execute(CommandContext ctxt) throws CommandException { logger.fine("getDataverseStorageSize called on " + dataset.getDisplayName()); - long total = 0L; if (dataset == null) { // should never happen - must indicate some data corruption in the database throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.listing.error"), this); } try { - total += ctxt.datasets().findStorageSize(dataset, countCachedFiles); + return ctxt.datasets().findStorageSize(dataset, countCachedFiles, mode, version); } catch (IOException ex) { throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.datasize.ioerror"), this); } - return total; + } + + @Override + public Map> getRequiredPermissions() { + // for data file check permission on owning dataset + return Collections.singletonMap("", + mode != null && mode.equals("storage") ? Collections.singleton(Permission.ManageDatasetPermissions) + : Collections.emptySet()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java index 05c62ea6a36..66e0f03055d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java @@ -59,7 +59,7 @@ public Long execute(CommandContext ctxt) throws CommandException { } try { - total += ctxt.datasets().findStorageSize(dataset, countCachedFiles); + total += ctxt.datasets().findStorageSize(dataset, countCachedFiles, "storage", null); } catch (IOException ex) { throw new CommandException(BundleUtil.getStringFromBundle("dataverse.datasize.ioerror"), this); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 43ff2d1e95d..11416791fbf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1321,6 +1321,15 @@ public void testDataSizeInDataverse() throws InterruptedException { // wait for it to ingest... assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile , UtilIT.sleepForLock(datasetId.longValue(), "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); // sleep(10000); + + Response publishDataversetResp = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); + publishDataversetResp.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response publishDatasetResp = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + //msg(publishDatasetResp.body().asString()); + publishDatasetResp.then().assertThat() + .statusCode(OK.getStatusCode()); // This is the magic number - the number of bytes in the 2 files uploaded // above, plus the size of the tab-delimited file generated by the ingest @@ -1336,11 +1345,16 @@ public void testDataSizeInDataverse() throws InterruptedException { magicControlString = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize"), magicSizeNumber); - Response datasetStorageSizeResponse = UtilIT.findDatasetStorageSize(datasetId.toString()); + Response datasetStorageSizeResponse = UtilIT.findDatasetStorageSize(datasetId.toString(), apiToken); datasetStorageSizeResponse.prettyPrint(); assertEquals(magicControlString, JsonPath.from(datasetStorageSizeResponse.body().asString()).getString("data.message")); + Response datasetDownloadSizeResponse = UtilIT.findDatasetDownloadSize(datasetId.toString()); + datasetDownloadSizeResponse.prettyPrint(); + + assertEquals(magicControlString, JsonPath.from(datasetDownloadSizeResponse.body().asString()).getString("data.message")); + } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7f090a41559..73612ed0169 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2304,8 +2304,15 @@ static Response findDataverseStorageSize(String dataverseId, String apiToken) { * @param apiToken * @return response */ - static Response findDatasetStorageSize(String datasetId) { + static Response findDatasetStorageSize(String datasetId, String apiToken) { return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/datasets/" + datasetId + "/storagesize"); } + + static Response findDatasetDownloadSize(String datasetId) { + return given() + .get("/api/datasets/" + datasetId + "/versions/:latest/downloadsize"); + } + }