Add IDataLoadJob.CrashAtEnd method

Fixes #1157

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ...
 
+### Added
+
+- Added `CrashAtEnd` system for DLE that allows Attachers to flag a load as a failure without halting execution [#1157](https://github.com/HicServices/RDMP/issues/1157)
+
 ## [7.0.12] - 2022-05-16
 
 ### Added

Rdmp.Core.Tests/DataLoad/Engine/Integration/ToMemoryDataLoadJob.cs

@@ -22,6 +22,10 @@ namespace Rdmp.Core.Tests.DataLoad.Engine.Integration
 {
     public class ToMemoryDataLoadJob : ToMemoryDataLoadEventListener, IDataLoadJob
     {
+        private List<NotifyEventArgs> _crashAtEnd = new ();
+        /// <inheritdoc/>
+        public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();
+
         public ToMemoryDataLoadJob(bool throwOnErrorEvents = true): base(throwOnErrorEvents)
         {
         }
@@ -64,5 +68,10 @@ public ColumnInfo[] GetAllColumns()
         {
             return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
         }
+        /// <inheritdoc/>
+        public void CrashAtEnd(NotifyEventArgs because)
+        {
+            _crashAtEnd.Add(because);
+        }
     }
 }

Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs

@@ -43,6 +43,10 @@ public class DataLoadJob : IDataLoadJob
         public HICDatabaseConfiguration Configuration { get; set; }
         public object Payload { get; set; }
 
+        private List<NotifyEventArgs> _crashAtEnd = new();
+
+        public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();
+
 
         private string _loggingTask;
 
@@ -202,5 +206,10 @@ public ColumnInfo[] GetAllColumns()
         {
             return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
         }
+
+        public void CrashAtEnd(NotifyEventArgs because)
+        {
+            _crashAtEnd.Add(because);
+        }
     }
 }

Rdmp.Core/DataLoad/Engine/Job/IDataLoadJob.cs

@@ -4,6 +4,7 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
+using System;
 using System.Collections.Generic;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
@@ -35,9 +36,15 @@ public interface IDataLoadJob : IDataLoadEventListener, IDisposeAfterDataLoad
         /// </summary>
         object Payload { get; set; }
 
+        /// <summary>
+        /// Collection of all calls to <see cref="CrashAtEnd"/>.  If there are any
+        /// of these at the end of the load they will be notified and a crash exit code will be 
+        /// returned (but otherwise the load will complete normally).
+        /// </summary>
+        IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages { get; }
         List<ITableInfo> RegularTablesToLoad { get; }
         List<ITableInfo> LookupTablesToLoad { get; }
-        
+
         IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; }
 
         void StartLogging();
@@ -60,5 +67,14 @@ public interface IDataLoadJob : IDataLoadEventListener, IDisposeAfterDataLoad
         /// </summary>
         /// <returns></returns>
         ColumnInfo[] GetAllColumns();
+
+        /// <summary>
+        /// Call you see that something has gone horribly wrong but want to keep going
+        /// with the load anyway.  Once the load has been completed these will crash
+        /// the process and result in a non 0 exit code.  Archiving and postload operations
+        /// will still occur.
+        /// </summary>
+        /// <param name="because"></param>
+        void CrashAtEnd(NotifyEventArgs because);
     }
 }

Rdmp.Core/DataLoad/Engine/Job/ThrowImmediatelyDataLoadJob.cs

@@ -70,6 +70,12 @@ public void CloseLogging()
 
         public object Payload { get; set; }
 
+        private List<NotifyEventArgs> _crashAtEnd = new();
+
+        /// <inheritdoc/>
+        public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();
+
+
         public void AddForDisposalAfterCompletion(IDisposeAfterDataLoad disposable)
         {
         }
@@ -99,5 +105,11 @@ public ColumnInfo[] GetAllColumns()
         {
             return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
         }
+
+        /// <inheritdoc/>
+        public void CrashAtEnd(NotifyEventArgs because)
+        {
+            _crashAtEnd.Add(because);
+        }
     }
 }

Rdmp.Core/DataLoad/Engine/LoadExecution/SingleJobExecution.cs

@@ -80,6 +80,20 @@ public ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellation
 
                 job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Completed job " + job.JobID));
 
+                if(job.CrashAtEndMessages.Count > 0)
+                {
+                    job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"There were {job.CrashAtEndMessages.Count} {nameof(IDataLoadJob.CrashAtEndMessages)} registered for job " + job.JobID));
+
+                    // pop the messages into the handler
+                    foreach (var m in job.CrashAtEndMessages)
+                    {
+                        job.OnNotify(job, m);  // depending on the listener these may break flow of control (e.g. 
+                    }
+
+                    // return failed (even if the messages are all warnings)
+                    return ExitCodeType.Error;                    
+                }
+
                 return ExitCodeType.Success;
             }
             catch (OperationCanceledException)

Rdmp.Core/DataLoad/Modules/Attachers/FlatFileAttacher.cs

@@ -40,6 +40,9 @@ public abstract class FlatFileAttacher : Attacher, IPluginAttacher
         [DemandsInitialization("Determines the behaviour of the system when no files are matched by FilePattern.  If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")]
         public bool SendLoadNotRequiredIfFileNotFound { get; set; }
 
+        [DemandsInitialization("If enabled then file(s) that could not be loaded are reported as warnings and the load only marked as failed after completion (including archiving etc)")]
+        public bool DelayLoadFailures { get; set; }
+
         public FlatFileAttacher() : base(true)
         {
 
@@ -94,8 +97,25 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken
             }
 
             foreach (var fileToLoad in filesToLoad)
-                LoadFile(table, fileToLoad, _dbInfo, timer, job,cancellationToken);
+            {
+                if(DelayLoadFailures)
+                {
+                    try
+                    {
+                        LoadFile(table, fileToLoad, _dbInfo, timer, job, cancellationToken);
+                    }
+                    catch (Exception ex)
+                    {
+                        job.CrashAtEnd(new NotifyEventArgs(ProgressEventType.Warning, $"Failed to load {fileToLoad}", ex));
+                    }
 
+                }
+                else
+                {
+                    LoadFile(table, fileToLoad, _dbInfo, timer, job, cancellationToken);
+                }
+            }
+
             timer.Stop();
 
             return ExitCodeType.Success;