Merge pull request #8271 from IQSS/8096-8205-ingest-messaging

improve messaging for ingest failures, file upload #8096

doc/release-notes/8096-8205-ingest-messaging.md

@@ -0,0 +1,36 @@
+Ingest and file upload messaging improvements
+
+On the pages for adding files on create or edit
+
+- Explain that all types are supported, link to tabular ingest guides
+- Show max files per upload #7227
+- Show global tabular ingest limit
+- Show per format tabular ingest limits
+- Move messaging from blue info block to regular text.
+
+Failed ingest messaging
+
+- Calm blue message that files are available in original format only
+- Click for a popup about ingest and specifics of why ingest failed
+
+Email notification for ingest success
+
+- Link to tabular ingest guides
+- Don't show the files that were just ingested
+
+Email notification for ingest failure
+
+- Link to tabular ingest guides
+- Say "incomplete" instead of "error"
+
+Email notification for mix of ingest success and failure
+
+- Looks just like the email for failure, no mention of files that had success
+
+In-app notification for ingest success
+
+- Reworded and links to guides
+
+In-app notification for ingest failure
+
+- Reworded and links to guides

doc/sphinx-guides/source/installation/config.rst

@@ -1747,7 +1747,12 @@ Notes:
 
 - For larger file upload sizes, you may need to configure your reverse proxy timeout. If using apache2 (httpd) with Shibboleth, add a timeout to the ProxyPass defined in etc/httpd/conf.d/ssl.conf (which is described in the :doc:`/installation/shibboleth` setup).
 
+:MultipleUploadFilesLimit
++++++++++++++++++++++++++
+
+This setting controls the number of files that can be uploaded through the UI at once. The default is 1000. It should be set to 1 or higher since 0 has no effect. To limit the number of files in a zip file, see ``:ZipUploadFilesLimit``.
 
+``curl -X PUT -d 500 http://localhost:8080/api/admin/settings/:MultipleUploadFilesLimit``
 
 :ZipDownloadLimit
 +++++++++++++++++

src/main/java/edu/harvard/iq/dataverse/DatasetPage.java

@@ -5805,4 +5805,9 @@ private boolean containsOnlyActivelyEmbargoedFiles(List<FileMetadata> selectedFi
         }
         return true;
     }
+
+    public String getIngestMessage() {
+        return BundleUtil.getStringFromBundle("file.ingestFailed.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion()));
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java

@@ -30,6 +30,7 @@
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.ingest.IngestUtil;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
+import edu.harvard.iq.dataverse.settings.Setting;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
@@ -170,6 +171,9 @@ public enum Referrer {
     private final Map<String, Boolean> datasetPermissionMap = new HashMap<>(); // { Permission human_name : Boolean }
 
     private Long maxFileUploadSizeInBytes = null;
+    private Long maxIngestSizeInBytes = null;
+    // CSV: 4.8 MB, DTA: 976.6 KB, XLSX: 5.7 MB, etc.
+    private String humanPerFormatTabularLimits = null;
     private Integer multipleUploadFilesLimit = null; 
 
     //MutableBoolean so it can be passed from DatasetPage, supporting DatasetPage.cancelCreate()
@@ -324,7 +328,35 @@ public boolean isUnlimitedUploadFileSize() {
 
         return this.maxFileUploadSizeInBytes == null;
     }
-
+
+    public Long getMaxIngestSizeInBytes() {
+        return maxIngestSizeInBytes;
+    }
+
+    public String getHumanMaxIngestSizeInBytes() {
+        return FileSizeChecker.bytesToHumanReadable(this.maxIngestSizeInBytes);
+    }
+
+    public String getHumanPerFormatTabularLimits() {
+        return humanPerFormatTabularLimits;
+    }
+
+    public String populateHumanPerFormatTabularLimits() {
+        String keyPrefix = ":TabularIngestSizeLimit:";
+        List<String> formatLimits = new ArrayList<>();
+        for (Setting setting : settingsService.listAll()) {
+            String name = setting.getName();
+            if (!name.startsWith(keyPrefix)) {
+                continue;
+            }
+            String tabularName = setting.getName().substring(keyPrefix.length());
+            String bytes = setting.getContent();
+            String humanReadableSize = FileSizeChecker.bytesToHumanReadable(Long.valueOf(bytes));
+            formatLimits.add(tabularName + ": " + humanReadableSize);
+        }
+        return String.join(", ", formatLimits);
+    }
+
     /*
         The number of files the GUI user is allowed to upload in one batch, 
         via drag-and-drop, or through the file select dialog. Now configurable 
@@ -463,6 +495,8 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
         selectedFiles = selectedFileMetadatasList;
 
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
+        this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
+        this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();
 
         logger.fine("done");
@@ -513,6 +547,8 @@ public String init() {
 
         clone = workingVersion.cloneDatasetVersion();   
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
+        this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
+        this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();        
 
         // -------------------------------------------
@@ -597,15 +633,6 @@ public String init() {
             setUpRsync();
         }
 
-        if (mode == FileEditMode.UPLOAD) {
-            if (settingsWrapper.getUploadMethodsCount() == 1){               
-                JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesSingle.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion())));
-            } else if (settingsWrapper.getUploadMethodsCount() > 1) {
-                JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.message.uploadFiles.label"), BundleUtil.getStringFromBundle("dataset.message.uploadFilesMultiple.message", Arrays.asList(systemConfig.getGuidesBaseUrl(), systemConfig.getGuidesVersion())));
-            }
-
-        }
-
         if (settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, false)){
             JH.addMessage(FacesMessage.SEVERITY_WARN, getBundleString("dataset.message.publicInstall"));
         }   

src/main/java/edu/harvard/iq/dataverse/FilePage.java

@@ -1100,4 +1100,9 @@ public String getEmbargoPhrase() {
             return BundleUtil.getStringFromBundle("embargoed.willbeuntil");
         }
     }
+
+    public String getIngestMessage() {
+        return BundleUtil.getStringFromBundle("file.ingestFailed.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion()));
+    }
+
 }

src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java

@@ -587,6 +587,8 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                         systemConfig.getDataverseSiteUrl(),
                         dataset.getGlobalIdString(),
                         dataset.getDisplayName(),
+                        systemConfig.getGuidesBaseUrl(),
+                        systemConfig.getGuidesVersion(),
                         comment
                 ));
 
@@ -598,6 +600,8 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                         systemConfig.getDataverseSiteUrl(),
                         dataset.getGlobalIdString(),
                         dataset.getDisplayName(),
+                        systemConfig.getGuidesBaseUrl(),
+                        systemConfig.getGuidesVersion(),
                         comment
                 ));
 

src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java

@@ -23,6 +23,7 @@
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 
 import java.sql.Timestamp;
 import java.time.Instant;
@@ -96,10 +97,13 @@ public void onMessage(Message message) {
                     if (ingestService.ingestAsTabular(datafile_id)) {
                         //Thread.sleep(10000);
                         logger.fine("Finished ingest job;");
-                        sbIngestedFiles.append(String.format("<li>%s</li>", datafile.getCurrentName()));
+                        // We used to list the successfully ingested files in the "success"
+                        // and "mixed success and failure" emails. Now we never list successfully
+                        // ingested files so this line is commented out.
+                        // sbIngestedFiles.append(String.format("<li>%s</li>", datafile.getCurrentName()));
                     } else {
                         logger.warning("Error occurred during ingest job for file id " + datafile_id + "!");
-                        sbIngestedFiles.append(String.format("<li>%s (Error)</li>", datafile.getCurrentName()));
+                        sbIngestedFiles.append(String.format("<li>%s</li>", datafile.getCurrentName()));
                         ingestWithErrors = true;
                     }
 
@@ -118,7 +122,7 @@ public void onMessage(Message message) {
 
                             ingestWithErrors = true;
 
-                            sbIngestedFiles.append(String.format("<li>%s (Error)</li>", datafile.getCurrentName()));
+                            sbIngestedFiles.append(String.format("<li>%s</li>", datafile.getCurrentName()));
 
                             datafile.SetIngestProblem();
                             IngestReport errorReport = new IngestReport();

src/main/java/propertyFiles/Bundle.properties

@@ -218,8 +218,8 @@ notification.workflowFailed=An external workflow run on {0} in {1} has failed. C
 notification.workflowSucceeded=An external workflow run on {0} in {1} has succeeded. Check your email and/or view the Dataset page which may have additional details.
 notification.statusUpdated=The status of dataset {0} has been updated to {1}.
 
-notification.ingestCompleted=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> ingest has successfully finished.
-notification.ingestCompletedWithErrors=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> ingest has finished with errors.
+notification.ingestCompleted=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> has one or more tabular files that completed the <a href="{2}/{3}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank" rel="noopener">tabular ingest process</a> and are available in archival formats.
+notification.ingestCompletedWithErrors=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> has one or more tabular files that are available but are not supported for <a href="{2}/{3}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files- Dataverse User Guide" target="_blank" rel="noopener">tabular ingest</a>.
 notification.generic.objectDeleted=The dataverse, dataset, or file for this notification has been deleted.
 notification.access.granted.dataverse=You have been granted the {0} role for {1}.
 notification.access.granted.dataset=You have been granted the {0} role for {1}.
@@ -230,8 +230,8 @@ notification.access.revoked.dataverse=You have been removed from a role in {0}.
 notification.access.revoked.dataset=You have been removed from a role in {0}.
 notification.access.revoked.datafile=You have been removed from a role in {0}.
 notification.checksumfail=One or more files in your upload failed checksum validation for dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a>. Please re-run the upload script. If the problem persists, please contact support.
-notification.ingest.completed=Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> ingest process has successfully finished.<br><br>Ingested files:{3}<br>
-notification.ingest.completedwitherrors=Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> ingest process has finished with errors.<br><br>Ingested files:{3}<br>
+notification.ingest.completed=Your Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> has one or more tabular files that completed the tabular ingest process. These files will be available for download in their original formats and other formats for enhanced archival purposes after you publish the dataset. The archival .tab files are displayed in the file table. Please see <a href="{3}/{4}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank" rel="noopener">the guides</a> for more information about ingest and support for tabular files.
+notification.ingest.completedwitherrors=Your Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> has one or more tabular files that have been uploaded successfully but are not supported for tabular ingest. After you publish the dataset, these files will not have additional archival features. Please see <a href="{3}/{4}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank" rel="noopener">the guides</a> for more information about ingest and support for tabular files.<br><br>Files with incomplete ingest:{5}
 notification.mail.import.filesystem=Dataset {2} ({0}/dataset.xhtml?persistentId={1}) has been successfully uploaded and verified.
 notification.import.filesystem=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> has been successfully uploaded and verified.
 notification.import.checksum=<a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a>, dataset had file checksums added via a batch job.
@@ -706,8 +706,8 @@ notification.email.create.account.subject={0}: Your account has been created
 notification.email.assign.role.subject={0}: You have been assigned a role
 notification.email.revoke.role.subject={0}: Your role has been revoked
 notification.email.verifyEmail.subject={0}: Verify your email address
-notification.email.ingestCompleted.subject={0}: Your ingest has successfully finished!
-notification.email.ingestCompletedWithErrors.subject={0}: Your ingest has finished with errors!
+notification.email.ingestCompleted.subject={0}: Dataset status
+notification.email.ingestCompletedWithErrors.subject={0}: Dataset status
 notification.email.greeting=Hello, \n
 notification.email.greeting.html=Hello, <br>
 # Bundle file editors, please note that "notification.email.welcome" is used in a unit test
@@ -1451,7 +1451,7 @@ dataset.subjectDisplay.title=Subject
 dataset.contact.tip=Use email button above to contact.
 dataset.asterisk.tip=Asterisks indicate required fields
 dataset.message.uploadFiles.label=Upload Dataset Files
-dataset.message.uploadFilesSingle.message=For more information about supported file formats, please refer to the <a href="{0}/{1}/user/dataset-management.html#file-handling-and-uploading" title="File Handling and Uploading - Dataverse User Guide" target="_blank">User Guide</a>.
+dataset.message.uploadFilesSingle.message=All file types are supported for upload and download in their original format. If you are uploading Excel, CSV, TSV, RData, Stata, or SPSS files, <a href="{0}/{1}/user/dataset-management.html##tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank">see the guides</a> for tabular support and limitations.
 dataset.message.uploadFilesMultiple.message=Multiple file upload/download methods are available for this dataset. Once you upload a file using one of these methods, your choice will be locked in for this dataset.
 dataset.message.editMetadata.label=Edit Dataset Metadata
 dataset.message.editMetadata.message=Add more metadata about this dataset to help others easily find it.
@@ -1575,6 +1575,9 @@ file.selectAllFiles=Select all {0} files in this dataset.
 file.dynamicCounter.filesPerPage=Files Per Page
 file.selectToAddBtn=Select Files to Add
 file.selectToAdd.tipLimit=File upload limit is {0} per file. 
+file.selectToAdd.tipMaxNumFiles=Maximum of {0} {0, choice, 0#files|1#file|2#files} per upload.
+file.selectToAdd.tipTabularLimit=<a href="{0}/{1}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank" rel="noopener">Tabular file ingest</a> is limited to {2}.
+file.selectToAdd.tipPerFileTabularLimit=Ingest is limited to the following file sizes based on their format: {0}.
 file.selectToAdd.tipMoreInformation=Select files or drag and drop into the upload widget.
 file.selectToAdd.dragdropMsg=Drag and drop files here.
 file.createUploadDisabled=Upload files using rsync via SSH. This method is recommended for large file transfers. The upload script will be available on the Upload Files page once you save this dataset.
@@ -1705,8 +1708,8 @@ file.spss-savEncoding.current=Current Selection:
 file.spss-porExtraLabels=Variable Labels
 file.spss-porExtraLabels.title=Upload an additional text file with extra variable labels.
 file.spss-porExtraLabels.selectToAddBtn=Select File to Add
-file.ingestFailed.header=Upload Completed with Errors
-file.ingestFailed.message=Tabular data ingest failed.
+file.ingestFailed.header=File available in original format only
+file.ingestFailed.message=<a href="{0}/{1}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank">Tabular ingest</a> was unsuccessful.
 file.downloadBtn.format.all=All File Formats + Information
 file.downloadBtn.format.tab=Tab-Delimited
 file.downloadBtn.format.original={0} (Original File Format)

src/main/webapp/dataverseuser.xhtml

@@ -317,13 +317,17 @@
                                                     <h:outputFormat value="#{bundle['notification.ingestCompleted']}" escape="false">
                                                         <f:param value="#{item.theObject.getGlobalId()}"/>
                                                         <f:param value="#{item.theObject.getDisplayName()}"/>
+                                                        <f:param value="#{settingsWrapper.guidesBaseUrl}"/>
+                                                        <f:param value="#{settingsWrapper.guidesVersion}"/>
                                                     </h:outputFormat>
                                                 </ui:fragment>
                                                 <ui:fragment rendered="#{item.type == 'INGESTCOMPLETEDWITHERRORS'}">
                                                     <i class="icon-dataset text-icon-inline text-muted"></i>
                                                     <h:outputFormat value="#{bundle['notification.ingestCompletedWithErrors']}" escape="false">
                                                         <f:param value="#{item.theObject.getGlobalId()}"/>
                                                         <f:param value="#{item.theObject.getDisplayName()}"/>
+                                                        <f:param value="#{settingsWrapper.guidesBaseUrl}"/>
+                                                        <f:param value="#{settingsWrapper.guidesVersion}"/>
                                                     </h:outputFormat>
                                                 </ui:fragment>
                                                 <ui:fragment rendered="#{item.type == 'WORKFLOW_FAILURE'}">