Merge pull request #8532 from lubitchv/8525-injest-optional-skip

8525 ingest optional skip

doc/release-notes/8525-ingest-optional-skip.md

@@ -0,0 +1 @@
+Tabular ingest can be skipped via API.

doc/sphinx-guides/source/api/native-api.rst

@@ -1301,6 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following:
 - A description of the file.
 - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset.
 - Whether or not the file is restricted.
+- Whether or not the file skips :doc:`tabular ingest </user/tabulardataingest/index>`. If the ``tabIngest`` parameter is not specified, it defaults to ``true``.
 
 Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide.
  
@@ -1315,13 +1316,13 @@ In the curl example below, all of the above are specified but they are optional.
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB
 
-  curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID"
+  curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB"
+  curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB"
 
 You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file.
 

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -3592,7 +3592,7 @@ public String save() {
                     // have been created in the dataset.
                     dataset = datasetService.find(dataset.getId());
 
-                    List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null);
+                    List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null, true);
                     newFiles.clear();
 
                     // and another update command:

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -1021,7 +1021,7 @@ public String save() {
             }
 
             // Try to save the NEW files permanently: 
-            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null);
+            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true);
 
             // reset the working list of fileMetadatas, as to only include the ones
             // that have been added to the version successfully: 

src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java

@@ -336,7 +336,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
                     throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\".");
                 } else {
 
-                    ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null);
+                    ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null, true);
 
                 }
             } else {

src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java

@@ -501,9 +501,11 @@ private boolean runAddReplaceFile(Dataset owner,
         if (!phase1Success){
             return false;
         }
-
-
-        return runAddReplacePhase2();
+        boolean tabIngest = true;
+        if (optionalFileParams != null) {
+            tabIngest = optionalFileParams.getTabIngest();
+        }
+        return runAddReplacePhase2(tabIngest);
 
     }
 
@@ -653,7 +655,7 @@ private boolean runAddReplacePhase1(Dataset owner,
 
 
     public boolean runReplaceFromUI_Phase2(){
-        return runAddReplacePhase2();
+        return runAddReplacePhase2(true);
     }
 
 
@@ -744,7 +746,7 @@ public boolean updateLabelDescriptionRestrictedFromUI(String label, String descr
      * 
      * @return 
      */
-    private boolean runAddReplacePhase2(){
+    private boolean runAddReplacePhase2(boolean tabIngest){
 
         if (this.hasError()){
             return false;   // possible to have errors already...
@@ -756,7 +758,7 @@ private boolean runAddReplacePhase2(){
         }
 
          msgt("step_060_addFilesViaIngestService");
-        if (!this.step_060_addFilesViaIngestService()){
+        if (!this.step_060_addFilesViaIngestService(tabIngest)){
             return false;
 
         }
@@ -1570,7 +1572,7 @@ private boolean step_055_loadOptionalFileParams(OptionalFileParams optionalFileP
         return true;
     }
 
-    private boolean step_060_addFilesViaIngestService(){
+    private boolean step_060_addFilesViaIngestService(boolean tabIngest){
 
         if (this.hasError()){
             return false;
@@ -1583,7 +1585,7 @@ private boolean step_060_addFilesViaIngestService(){
         }
 
         int nFiles = finalFileList.size();
-        finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace);
+        finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace, tabIngest);
 
         if (nFiles != finalFileList.size()) {
             if (nFiles == 1) {
@@ -2244,4 +2246,4 @@ public String getFileName()
     1) Recovery from adding same file and duplicate being found
         - draft ok
         - published verion - nope
-*/
+*/

src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java

@@ -63,6 +63,9 @@ public class OptionalFileParams {
 
     private boolean restrict = false;
     public static final String RESTRICT_ATTR_NAME = "restrict";
+
+    private boolean tabIngest = true;
+    public static final String TAB_INGEST_ATTR_NAME = "tabIngest";
 
     private String storageIdentifier;
     public static final String STORAGE_IDENTIFIER_ATTR_NAME = "storageIdentifier";
@@ -173,7 +176,15 @@ public void setRestriction(boolean restrict){
     public boolean getRestriction(){
         return this.restrict;
     }
-
+
+    public void setTabIngest(boolean tabIngest) {
+        this.tabIngest = tabIngest;
+    }
+
+    public boolean getTabIngest() {
+        return this.tabIngest;
+    }
+
     public boolean hasCategories(){
         if ((categories == null)||(this.categories.isEmpty())){
             return false;
@@ -347,6 +358,14 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{
 
             this.restrict = Boolean.valueOf(jsonObj.get(RESTRICT_ATTR_NAME).getAsString());
         }
+
+        // -------------------------------
+        // get tabIngest as boolean
+        // -------------------------------
+        if ((jsonObj.has(TAB_INGEST_ATTR_NAME)) && (!jsonObj.get(TAB_INGEST_ATTR_NAME).isJsonNull())){
+
+            this.tabIngest = Boolean.valueOf(jsonObj.get(TAB_INGEST_ATTR_NAME).getAsString());
+        }
 
         // -------------------------------
         // get storage identifier as string

src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java

@@ -154,8 +154,11 @@ public class IngestServiceBean {
     // attached to the Dataset via some cascade path (for example, via 
     // DataFileCategory objects, if any were already assigned to the files). 
     // It must be called before we attempt to permanently save the files in 
-    // the database by calling the Save command on the dataset and/or version. 
-	public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version, List<DataFile> newFiles, DataFile fileToReplace) {
+    // the database by calling the Save command on the dataset and/or version.
+    public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
+                                                   List<DataFile> newFiles,
+                                                   DataFile fileToReplace,
+                                                   boolean tabIngest) {
 		List<DataFile> ret = new ArrayList<>();
 
 		if (newFiles != null && newFiles.size() > 0) {
@@ -327,7 +330,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version, List<Data
 					String fileName = fileMetadata.getLabel();
 
 					boolean metadataExtracted = false;
-					if (FileUtil.canIngestAsTabular(dataFile)) {
+					if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
 						/*
 						 * Note that we don't try to ingest the file right away - instead we mark it as
 						 * "scheduled for ingest", then at the end of the save process it will be queued

src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java

@@ -2586,5 +2586,7 @@ public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException {
                 .body("data.latestVersion.files[0].directoryLabel", equalTo("code"));
 
     }
+
+
 
 }

src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java

@@ -3,8 +3,9 @@
 import com.jayway.restassured.RestAssured;
 import com.jayway.restassured.response.Response;
 import java.util.logging.Logger;
-import org.junit.BeforeClass;
+
 import org.junit.Test;
+import org.junit.BeforeClass;
 import com.jayway.restassured.path.json.JsonPath;
 import com.jayway.restassured.path.xml.XmlPath;
 import static edu.harvard.iq.dataverse.api.AccessIT.apiToken;
@@ -39,11 +40,10 @@
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.CoreMatchers.nullValue;
 import org.hamcrest.Matchers;
-import org.junit.AfterClass;
+
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import org.junit.Ignore;
 
 public class FilesIT {
 
@@ -1784,4 +1784,58 @@ public void testRange() throws IOException {
 
     }
 
+    @Test
+    public void testAddFileToDatasetSkipTabIngest() throws IOException, InterruptedException {
+
+        Response createUser = UtilIT.createRandomUser();
+        assertEquals(200, createUser.getStatusCode());
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        assertEquals(201, createDataverseResponse.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        assertEquals(201, createDatasetResponse.getStatusCode());
+        Integer datasetIdInt = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        String pathToFile = "src/test/resources/sav/dct.sav";
+        String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}";
+        Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken);
+        logger.info(r.prettyPrint());
+        assertEquals(200, r.getStatusCode());
+
+        assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION));
+
+        Long dataFileId = JsonPath.from(r.body().asString()).getLong("data.files[0].dataFile.id");
+        Response fileMeta = UtilIT.getDataFileMetadataDraft(dataFileId, apiToken);
+        String label = JsonPath.from(fileMeta.body().asString()).getString("label");
+        assertEquals("dct.sav", label);
+
+        pathToFile = "src/test/resources/sav/frequency-test.sav";
+        jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\"  }";
+        Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken);
+        logger.info(rTabIngest.prettyPrint());
+        assertEquals(200, rTabIngest.getStatusCode());
+
+        assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION));
+
+        Long ingDataFileId = JsonPath.from(rTabIngest.body().asString()).getLong("data.files[0].dataFile.id");
+        Response ingFileMeta = UtilIT.getDataFileMetadataDraft(ingDataFileId, apiToken);
+        String ingLabel = JsonPath.from(ingFileMeta.body().asString()).getString("label");
+        assertEquals("frequency-test.tab", ingLabel);
+
+        //cleanup
+        Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken);
+        assertEquals(200, destroyDatasetResponse.getStatusCode());
+
+        Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
+        assertEquals(200, deleteDataverseResponse.getStatusCode());
+
+        Response deleteUserResponse = UtilIT.deleteUser(username);
+        assertEquals(200, deleteUserResponse.getStatusCode());
+
+    }
+
 }