Dataset files API extension for file display data with pagination and sorting

doc/release-notes/9692-files-api-extension.md

@@ -0,0 +1,8 @@
+The following API endpoints have been added:
+
+- /api/files/{id}/guestbookResponses/count
+- /api/files/{id}/canBeDownloaded
+- /api/files/{id}/thumbnailClass
+- /api/files/{id}/dataTables
+
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination and ordering

src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java

@@ -49,7 +49,23 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
     private static final Logger logger = Logger.getLogger(DatasetVersionServiceBean.class.getCanonicalName());
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
-
+
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
+            + " WHERE fm.datasetVersion.id=:datasetVersionId"
+            + " ORDER BY fm.label";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = dvo.id"
+            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.filesize";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.contentType";
+
     @EJB
     DatasetServiceBean datasetService;
 
@@ -150,7 +166,19 @@ public DatasetVersion getDatasetVersion(){
             return this.datasetVersionForResponse;
         }                
     } // end RetrieveDatasetVersionResponse
-
+
+    /**
+     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ,
+        NameZA,
+        Newest,
+        Oldest,
+        Size,
+        Type
+    }
+
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1225,4 +1253,50 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit for pagination, can be null
+     * @param offset for pagination, can be null
+     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list of the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
+        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
+                .setParameter("datasetVersionId", datasetVersion.getId());
+        if (limit != null) {
+            query.setMaxResults(limit);
+        }
+        if (offset != null) {
+            query.setFirstResult(offset);
+        }
+        return query.getResultList();
+    }
+
+    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
+        String queryString;
+        switch (orderCriteria) {
+            case NameZA:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
+                break;
+            case Newest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
+                break;
+            case Oldest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
+                break;
+            case Size:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
+                break;
+            case Type:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
+                break;
+            default:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
+                break;
+        }
+        return queryString;
+    }
 } // end class

src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java

@@ -8,6 +8,7 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookResponseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand;
@@ -18,7 +19,6 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import java.io.IOException;
@@ -36,7 +36,6 @@
 import javax.inject.Named;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
-import javax.persistence.Query;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletResponse;
 
@@ -572,5 +571,15 @@ public String getDirectStorageLocatrion(String storageLocation) {
 
         return null; 
     }
-
+
+    /**
+     *  Checks if the DataverseRequest, which contains IP Groups, has permission to download the file
+     *
+     * @param dataverseRequest the DataverseRequest
+     * @param dataFile the DataFile to check permissions
+     * @return boolean
+     */
+    public boolean canDownloadFile(DataverseRequest dataverseRequest, DataFile dataFile) {
+        return permissionService.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
+    }
 }

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -31,7 +31,7 @@
 import edu.harvard.iq.dataverse.RoleAssignment;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
-import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
+
 import static edu.harvard.iq.dataverse.api.Datasets.handleVersion;
 
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
@@ -121,8 +121,6 @@
 import javax.ws.rs.core.StreamingOutput;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 import java.net.URISyntaxException;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import javax.json.JsonObjectBuilder;
 import javax.ws.rs.RedirectionException;
 import javax.ws.rs.ServerErrorException;
@@ -1946,5 +1944,22 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin
             throw new BadRequestException(); 
         }
         return redirectUri;
-    }   
+    }
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userPermissions")
+    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        User requestUser = getRequestUser(crc);
+        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
+        return ok(jsonObjectBuilder);
+    }
 }

src/main/java/edu/harvard/iq/dataverse/api/Datasets.java

@@ -493,9 +493,17 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response( req -> ok( jsonFileMetadatas(
-                         getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getFileMetadatas())), getRequestUser(crc));
+    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response( req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            try {
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
+            }
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
+        }, getRequestUser(crc));
     }
 
     @GET

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -3,6 +3,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetLock;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
@@ -11,7 +12,9 @@
 import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.FileDownloadServiceBean;
 import edu.harvard.iq.dataverse.FileMetadata;
+import edu.harvard.iq.dataverse.GuestbookResponseServiceBean;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
@@ -73,6 +76,8 @@
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT;
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import javax.ws.rs.core.UriInfo;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -102,7 +107,11 @@ public class Files extends AbstractApiBean {
     SettingsServiceBean settingsService;
     @Inject
     MakeDataCountLoggingServiceBean mdcLogService;
-
+    @Inject
+    GuestbookResponseServiceBean guestbookResponseService;
+    @Inject
+    DataFileServiceBean dataFileServiceBean;
+
     private static final Logger logger = Logger.getLogger(Files.class.getName());
 
 
@@ -818,4 +827,37 @@ public Response getExternalToolFMParams(@Context ContainerRequestContext crc, @P
     public Response getFixityAlgorithm() {
         return ok(systemConfig.getFileFixityChecksumAlgorithm().toString());
     }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/guestbookResponses/count")
+    public Response getCountGuestbookResponses(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
+        }, getRequestUser(crc));
+    }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/thumbnailClass")
+    public Response getFileThumbnailClass(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(dataFileServiceBean.getFileThumbnailClass(dataFile));
+        }, getRequestUser(crc));
+    }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/dataTables")
+    public Response getFileDataTables(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            if (!dataFile.isTabularData()) {
+                return error(BAD_REQUEST, "This operation is only available for tabular files.");
+            }
+            return ok(jsonDT(dataFile.getDataTables()));
+        }, getRequestUser(crc));
+    }
 }

src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java

@@ -692,7 +692,8 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue()))
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
                 .add("tabularTags", getTabularFileTags(df))
-                .add("creationDate", df.getCreateDateFormattedYYYYMMDD());
+                .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the
@@ -759,7 +760,7 @@ public static JsonObjectBuilder json(DataVariable dv) {
             .add("variableMetadata",jsonVarMetadata(dv.getVariableMetadatas()))
             .add("invalidRanges", dv.getInvalidRanges().isEmpty() ? null : JsonPrinter.jsonInvalidRanges(dv.getInvalidRanges()))
             .add("summaryStatistics", dv.getSummaryStatistics().isEmpty() ? null : JsonPrinter.jsonSumStat(dv.getSummaryStatistics()))
-            .add("variableCategories", dv.getCategories().isEmpty() ? null : JsonPrinter.jsonCatStat(dv.getCategories())) 
+            .add("variableCategories", dv.getCategories().isEmpty() ? null : JsonPrinter.jsonCatStat(dv.getCategories()))
             ;
     }
 

src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java

@@ -23,8 +23,12 @@
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.HashMap;
-import static javax.ws.rs.core.Response.Status.OK;
+
 import org.hamcrest.collection.IsMapContaining;
+
+import javax.json.Json;
+
+import static javax.ws.rs.core.Response.Status.*;
 import static junit.framework.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
@@ -629,7 +633,39 @@ public void testZipUploadAndDownload() throws IOException {
         System.out.println("MD5 checksums of the unzipped file streams are correct.");
 
         System.out.println("Zip upload-and-download round trip test: success!");
-
     }
 
+    @Test
+    public void testGetUserPermissionsOnFile() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert user permissions on file
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken);
+
+        getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile");
+        assertTrue(canDownloadFile);
+        boolean canEditOwnerDataset = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canEditOwnerDataset");
+        assertTrue(canEditOwnerDataset);
+
+        // Call with invalid file id
+        Response getUserPermissionsOnFileInvalidIdResponse = UtilIT.getUserPermissionsOnFile("testInvalidId", apiToken);
+        getUserPermissionsOnFileInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+    }
 }