Merge pull request #6271 from CIMMYT/6214-User-Notification-Following…

…-Ingest

6214 user notification following ingest

doc/sphinx-guides/source/user/account.rst

@@ -158,6 +158,7 @@ You will receive a notification when:
 - You've created your account
 - You've created a dataverse or added a dataset
 - Another Dataverse user has requested access to a restricted file in one of your datasets
+- A file in one of your datasets has finished the ingest process
 
 Notifications will only be emailed one time even if you haven't read the notification on the Dataverse site.
 

src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java

@@ -476,7 +476,7 @@ public boolean requestAccess(Long fileId) {
 
     public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) {
         permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> {
-            userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor);
+            userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
         });
 
     }    

src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java

@@ -114,6 +114,11 @@ public void sendMail(String host, String reply, String to, String subject, Strin
     private Session session;
 
     public boolean sendSystemEmail(String to, String subject, String messageText) {
+        return sendSystemEmail(to, subject, messageText, false);
+    }
+
+    public boolean sendSystemEmail(String to, String subject, String messageText, boolean isHtmlContent) {
+
 
         boolean sent = false;
         String rootDataverseName = dataverseService.findRootDataverse().getName();
@@ -136,7 +141,12 @@ public boolean sendSystemEmail(String to, String subject, String messageText) {
                 }
                 msg.setRecipients(Message.RecipientType.TO, recipients);
                 msg.setSubject(subject, charset);
-                msg.setText(body, charset);
+                if (isHtmlContent) {
+                    msg.setText(body, charset, "html");
+                } else {
+                    msg.setText(body, charset);
+                }
+
                 try {
                     Transport.send(msg, recipients);
                     sent = true;
@@ -215,11 +225,15 @@ public Boolean sendNotificationEmail(UserNotification notification){
 
 
     public Boolean sendNotificationEmail(UserNotification notification, String comment) {
-        return sendNotificationEmail(notification, comment, null);
+        return sendNotificationEmail(notification, comment, null, false);
+    }
+
+    public Boolean sendNotificationEmail(UserNotification notification, String comment, boolean isHtmlContent) {
+        return sendNotificationEmail(notification, comment, null, isHtmlContent);
     }
-    
-    
-    public Boolean sendNotificationEmail(UserNotification notification, String comment, AuthenticatedUser requestor){  
+
+
+    public Boolean sendNotificationEmail(UserNotification notification, String comment, AuthenticatedUser requestor, boolean isHtmlContent){
 
         boolean retval = false;
         String emailAddress = getUserEmailAddress(notification);
@@ -230,7 +244,7 @@ public Boolean sendNotificationEmail(UserNotification notification, String comme
                String rootDataverseName = dataverseService.findRootDataverse().getName();
                String subjectText = MailUtil.getSubjectTextBasedOnNotification(notification, rootDataverseName, objectOfNotification);
                if (!(messageText.isEmpty() || subjectText.isEmpty())){
-                    retval = sendSystemEmail(emailAddress, subjectText, messageText); 
+                   retval = sendSystemEmail(emailAddress, subjectText, messageText, isHtmlContent);
                } else {
                    logger.warning("Skipping " + notification.getType() +  " notification, because couldn't get valid message");
                }
@@ -530,6 +544,29 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 String message = BundleUtil.getStringFromBundle("notification.email.apiTokenGenerated", Arrays.asList(
                         userNotification.getUser().getFirstName(), userNotification.getUser().getFirstName() ));
                 return message;
+
+            case INGESTCOMPLETED:
+                dataset = (Dataset) targetObject;
+
+                String ingestedCompletedMessage = messageText + BundleUtil.getStringFromBundle("notification.ingest.completed", Arrays.asList(
+                        systemConfig.getDataverseSiteUrl(),
+                        dataset.getGlobalIdString(),
+                        dataset.getDisplayName(),
+                        comment
+                ));
+
+                return ingestedCompletedMessage;
+            case INGESTCOMPLETEDWITHERRORS:
+                dataset = (Dataset) targetObject;
+
+                String ingestedCompletedWithErrorsMessage = messageText + BundleUtil.getStringFromBundle("notification.ingest.completedwitherrors", Arrays.asList(
+                        systemConfig.getDataverseSiteUrl(),
+                        dataset.getGlobalIdString(),
+                        dataset.getDisplayName(),
+                        comment
+                ));
+
+                return ingestedCompletedWithErrorsMessage;
         }
 
         return "";
@@ -572,6 +609,9 @@ private Object getObjectOfNotification (UserNotification userNotification){
                 return versionService.find(userNotification.getObjectId());
             case APIGENERATED:
                 return userNotification.getUser();
+            case INGESTCOMPLETED:
+            case INGESTCOMPLETEDWITHERRORS:
+                return datasetService.find(userNotification.getObjectId());
 
         }
         return null;

src/main/java/edu/harvard/iq/dataverse/UserNotification.java

@@ -27,7 +27,7 @@
 
 public class UserNotification implements Serializable {
     public enum Type {
-        ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, MAPLAYERUPDATED, MAPLAYERDELETEFAILED, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED
+        ASSIGNROLE, REVOKEROLE, CREATEDV, CREATEDS, CREATEACC, MAPLAYERUPDATED, MAPLAYERDELETEFAILED, SUBMITTEDDS, RETURNEDDS, PUBLISHEDDS, REQUESTFILEACCESS, GRANTFILEACCESS, REJECTFILEACCESS, FILESYSTEMIMPORT, CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS
     };
 
     private static final long serialVersionUID = 1L;

src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java

@@ -81,25 +81,29 @@ public UserNotification save(UserNotification userNotification) {
 
     public void delete(UserNotification userNotification) {
         em.remove(em.merge(userNotification));
-    }    
+    }
 
     public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId) {
         sendNotification(dataverseUser, sendDate, type, objectId, "");
     }
-    
+
     public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment) {
-        sendNotification(dataverseUser, sendDate, type, objectId, comment, null);
+        sendNotification(dataverseUser, sendDate, type, objectId, comment, null, false);
     }
-
-    public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment, AuthenticatedUser requestor) {
+
+    public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment, boolean isHtmlContent) {
+        sendNotification(dataverseUser, sendDate, type, objectId, comment, null, isHtmlContent);
+    }
+
+    public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId, String comment, AuthenticatedUser requestor, boolean isHtmlContent) {
         UserNotification userNotification = new UserNotification();
         userNotification.setUser(dataverseUser);
         userNotification.setSendDate(sendDate);
         userNotification.setType(type);
         userNotification.setObjectId(objectId);
         userNotification.setRequestor(requestor);
-        
-        if (mailService.sendNotificationEmail(userNotification, comment, requestor)) {
+
+        if (mailService.sendNotificationEmail(userNotification, comment, requestor, isHtmlContent)) {
             logger.fine("email was sent");
             userNotification.setEmailed(true);
             save(userNotification);

src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java

@@ -513,6 +513,11 @@ public void displayNotification() {
                 case APIGENERATED:
                     userNotification.setTheObject(userNotification.getUser());
                     break;
+
+                case INGESTCOMPLETED:
+                case INGESTCOMPLETEDWITHERRORS:
+                    userNotification.setTheObject(datasetService.find(userNotification.getObjectId()));
+                    break;
             }
 
             userNotification.setDisplayAsRead(userNotification.isReadNotification());

src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SubmitDatasetForReviewCommand.java

@@ -61,7 +61,7 @@ public Dataset save(CommandContext ctxt) throws CommandException {
 
         List<AuthenticatedUser> authUsers = ctxt.permissions().getUsersWithPermissionOn(Permission.PublishDataset, savedDataset);
         for (AuthenticatedUser au : authUsers) {
-            ctxt.notifications().sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.SUBMITTEDDS, savedDataset.getLatestVersion().getId(), "", requestor);
+            ctxt.notifications().sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.SUBMITTEDDS, savedDataset.getLatestVersion().getId(), "", requestor, false);
         }
 
         //  TODO: What should we do with the indexing result? Print it to the log?

src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessage.java

@@ -44,14 +44,21 @@ public IngestMessage(int messageLevel)  {
         this.messageLevel = messageLevel;
         datafile_ids = new ArrayList<Long>();
     }
-
+
+    public IngestMessage(int messageLevel, Long authenticatedUserId) {
+        this.messageLevel = messageLevel;
+        this.authenticatedUserId = authenticatedUserId;
+        datafile_ids = new ArrayList<Long>();
+    }
+
     private int messageLevel = INGEST_MESAGE_LEVEL_INFO;
 
     private Long datasetId;
     private Long datasetVersionId;
     private String versionNote;
     private String datasetVersionNumber;
     private List<Long> datafile_ids;
+    private Long authenticatedUserId;
 
     public String getVersionNote() {
         return versionNote;
@@ -112,4 +119,8 @@ public void setFileIds(List<Long> file_ids) {
     public void addFileId(Long file_id) {
         datafile_ids.add(file_id);
     }
+
+    public Long getAuthenticatedUserId() {
+        return authenticatedUserId;
+    }
 }

src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java

@@ -20,12 +20,12 @@
 
 package edu.harvard.iq.dataverse.ingest;
 
-import edu.harvard.iq.dataverse.DatasetServiceBean;
-import edu.harvard.iq.dataverse.DataFileServiceBean;
-import edu.harvard.iq.dataverse.DataFile; 
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetLock;
+import edu.harvard.iq.dataverse.*;
+import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 
+import java.sql.Timestamp;
+import java.time.Instant;
 import java.util.Iterator;
 import java.util.logging.Logger;
 import javax.ejb.ActivationConfigProperty;
@@ -50,7 +50,9 @@ public class IngestMessageBean implements MessageListener {
     private static final Logger logger = Logger.getLogger(IngestMessageBean.class.getCanonicalName());
     @EJB DatasetServiceBean datasetService;
     @EJB DataFileServiceBean datafileService;
-    @EJB IngestServiceBean ingestService; 
+    @EJB IngestServiceBean ingestService;
+    @EJB UserNotificationServiceBean userNotificationService;
+    @EJB AuthenticationServiceBean authenticationServiceBean;
 
 
     public IngestMessageBean() {
@@ -60,26 +62,41 @@ public IngestMessageBean() {
     public void onMessage(Message message) {
         IngestMessage ingestMessage = null;
 
-        Long datafile_id = null; 
+        Long datafile_id = null;
+        AuthenticatedUser authenticatedUser = null;
 
         try {
             ObjectMessage om = (ObjectMessage) message;
             ingestMessage = (IngestMessage) om.getObject();
 
+            authenticatedUser = authenticationServiceBean.findByID(ingestMessage.getAuthenticatedUserId());
+
             Iterator iter = ingestMessage.getFileIds().iterator();
-            datafile_id = null; 
+            datafile_id = null;
+
+            boolean ingestWithErrors = false;
+
+            StringBuilder sbIngestedFiles = new StringBuilder();
+            sbIngestedFiles.append("<ul>");
 
             while (iter.hasNext()) {
                 datafile_id = (Long) iter.next();
 
                 logger.fine("Start ingest job;");
                 try {
+
+                    DataFile datafile = datafileService.find(datafile_id);
+
                     if (ingestService.ingestAsTabular(datafile_id)) {
                         //Thread.sleep(10000);
                         logger.fine("Finished ingest job;");
+                        sbIngestedFiles.append(String.format("<li>%s</li>", datafile.getCurrentName()));
                     } else {
                         logger.warning("Error occurred during ingest job for file id " + datafile_id + "!");
+                        sbIngestedFiles.append(String.format("<li>%s (Error)</li>", datafile.getCurrentName()));
+                        ingestWithErrors = true;
                     }
+
                 } catch (Exception ex) {
                     //ex.printStackTrace();
                     // TODO: 
@@ -92,6 +109,11 @@ public void onMessage(Message message) {
                         logger.fine("looking up datafile for id " + datafile_id);
                         DataFile datafile = datafileService.find(datafile_id);
                         if (datafile != null) {
+
+                            ingestWithErrors = true;
+
+                            sbIngestedFiles.append(String.format("<li>%s (Error)</li>", datafile.getCurrentName()));
+
                             datafile.SetIngestProblem();
                             IngestReport errorReport = new IngestReport();
                             errorReport.setFailure();
@@ -117,6 +139,10 @@ public void onMessage(Message message) {
                     }
                 }
             }
+
+            sbIngestedFiles.append("</ul>");
+
+            Long objectId = null;
 
             // Remove the dataset lock: 
             // (note that the assumption here is that all of the datafiles
@@ -125,11 +151,22 @@ public void onMessage(Message message) {
                 DataFile datafile = datafileService.find(datafile_id);
                 if (datafile != null) {
                     Dataset dataset = datafile.getOwner();
+                    objectId = dataset.getId();
                     if (dataset != null && dataset.getId() != null) {
                         datasetService.removeDatasetLocks(dataset, DatasetLock.Reason.Ingest);
                     }
                 } 
-            } 
+            }
+
+            userNotificationService.sendNotification(
+                    authenticatedUser,
+                    Timestamp.from(Instant.now()),
+                    !ingestWithErrors ? UserNotification.Type.INGESTCOMPLETED : UserNotification.Type.INGESTCOMPLETEDWITHERRORS,
+                    objectId,
+                    sbIngestedFiles.toString(),
+                    true
+            );
+
 
         } catch (JMSException ex) {
             ex.printStackTrace(); // error in getting object from message; can't send e-mail

src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java

@@ -475,7 +475,7 @@ public int compare(DataFile d1, DataFile d2) {
                 }
             });
 
-            ingestMessage = new IngestMessage(IngestMessage.INGEST_MESAGE_LEVEL_INFO);
+            ingestMessage = new IngestMessage(IngestMessage.INGEST_MESAGE_LEVEL_INFO, user.getId());
             for (int i = 0; i < count; i++) {
                 ingestMessage.addFileId(scheduledFilesArray[i].getId());
             }

src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java

@@ -75,6 +75,10 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti
                 return BundleUtil.getStringFromBundle("notification.email.verifyEmail.subject", rootDvNameAsList);
             case APIGENERATED:
                 return BundleUtil.getStringFromBundle("notification.email.apiTokenGenerated.subject",  rootDvNameAsList);
+            case INGESTCOMPLETED:
+                return BundleUtil.getStringFromBundle("notification.email.ingestCompleted.subject", rootDvNameAsList);
+            case INGESTCOMPLETEDWITHERRORS:
+                return BundleUtil.getStringFromBundle("notification.email.ingestCompletedWithErrors.subject", rootDvNameAsList);
         }
         return "";
     }

src/main/java/propertyFiles/Bundle.properties

@@ -187,6 +187,8 @@ notification.dataset.management.title=Dataset Management - Dataset User Guide
 notification.wasSubmittedForReview={0} was submitted for review to be published in {1}. Don''t forget to publish it or send it back to the contributor, {2} ({3})\!
 notification.wasReturnedByReviewer={0} was returned by the curator of {1}.
 notification.wasPublished={0} was published in {1}.
+notification.ingestCompleted=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> ingest has successfully finished.
+notification.ingestCompletedWithErrors=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> ingest has finished with errors.
 notification.worldMap.added={0}, dataset had WorldMap layer data added to it.
 notification.maplayer.deletefailed=Failed to delete the map layer associated with the restricted file {0} from WorldMap. Please try again, or contact WorldMap and/or Dataverse support. (Dataset: {1})
 notification.generic.objectDeleted=The dataverse, dataset, or file for this notification has been deleted.
@@ -199,6 +201,8 @@ notification.access.revoked.dataverse=You have been removed from a role in {0}.
 notification.access.revoked.dataset=You have been removed from a role in {0}.
 notification.access.revoked.datafile=You have been removed from a role in {0}.
 notification.checksumfail=One or more files in your upload failed checksum validation for dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a>. Please re-run the upload script. If the problem persists, please contact support.
+notification.ingest.completed=Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> ingest process has successfully finished.<br><br>Ingested files:{3}<br>
+notification.ingest.completedwitherrors=Dataset <a href="{0}/dataset.xhtml?persistentId={1}" title="{2}">{2}</a> ingest process has finished with errors.<br><br>Ingested files:{3}<br>
 notification.mail.import.filesystem=Dataset {2} ({0}/dataset.xhtml?persistentId={1}) has been successfully uploaded and verified.
 notification.import.filesystem=Dataset <a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a> has been successfully uploaded and verified.
 notification.import.checksum=<a href="/dataset.xhtml?persistentId={0}" title="{1}">{1}</a>, dataset had file checksums added via a batch job.
@@ -636,6 +640,8 @@ notification.email.create.account.subject={0}: Your account has been created
 notification.email.assign.role.subject={0}: You have been assigned a role
 notification.email.revoke.role.subject={0}: Your role has been revoked
 notification.email.verifyEmail.subject={0}: Verify your email address
+notification.email.ingestCompleted.subject={0}: Your ingest has successfully finished!
+notification.email.ingestCompletedWithErrors.subject={0}: Your ingest has finished with errors!
 notification.email.greeting=Hello, \n
 # Bundle file editors, please note that "notification.email.welcome" is used in a unit test
 notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance.