RDMP-15 Use .bak files as Data Loads (#1656)

* basic ui triggers
* working dl

---------

Co-authored-by: James A Sutherland <j@sutherland.pw>

Rdmp.Core/CommandExecution/AtomicCommandFactory.cs

@@ -545,6 +545,8 @@ public IEnumerable<IAtomicCommand> CreateCommands(object o)
                 lsn.LoadMetadata, lsn.LoadStage);
             yield return new ExecuteCommandCreateNewFileBasedProcessTask(_activator, ProcessTaskType.Executable,
                 lsn.LoadMetadata, lsn.LoadStage);
+            yield return new ExecuteCommandCreateNewFileBasedProcessTask(_activator, ProcessTaskType.SQLBakFile,
+                lsn.LoadMetadata, lsn.LoadStage);
         }
 
         if (Is(o, out LoadDirectoryNode ldn))

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewFileBasedProcessTask.cs

@@ -6,6 +6,7 @@
 
 using System;
 using System.IO;
+using System.Linq;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.Icons.IconOverlays;
@@ -41,8 +42,9 @@ public ExecuteCommandCreateNewFileBasedProcessTask(IBasicActivateItems activator
             SetImpossible("Could not construct LoadDirectory");
         }
 
-        if (taskType is not (ProcessTaskType.SQLFile or ProcessTaskType.Executable))
-            SetImpossible("Only SQLFile and Executable task types are supported by this command");
+        ProcessTaskType[] AcceptedProcessTaskTypes = { ProcessTaskType.SQLFile, ProcessTaskType.Executable, ProcessTaskType.SQLBakFile };
+        if (!AcceptedProcessTaskTypes.Contains(taskType))
+            SetImpossible("Only SQLFile, SqlBakFile and Executable task types are supported by this command");
 
         if (!ProcessTask.IsCompatibleStage(taskType, loadStage))
             SetImpossible($"You cannot run {taskType} in {loadStage}");
@@ -56,26 +58,37 @@ public override void Execute()
 
         if (_file == null)
         {
-            if (_taskType == ProcessTaskType.SQLFile)
+            if (_taskType == ProcessTaskType.SQLBakFile)
             {
-                if (BasicActivator.TypeText("Enter a name for the SQL file", "File name", 100, "myscript.sql",
-                        out var selected, false))
-                {
-                    var target = Path.Combine(_loadDirectory.ExecutablesPath.FullName, selected);
-
-                    if (!target.EndsWith(".sql"))
-                        target += ".sql";
+                if (!BasicActivator.TypeText("Enter a name for the SQL Bak file", "File name", 100, "database.bak",
+                       out var selected, false)) return;
 
-                    //create it if it doesn't exist
-                    if (!File.Exists(target))
-                        File.WriteAllText(target, "/*todo Type some SQL*/");
+                var target = Path.Combine(_loadDirectory.ExecutablesPath.FullName, selected);
 
-                    _file = new FileInfo(target);
-                }
-                else
+                if (!File.Exists(target))
                 {
-                    return; //user cancelled
+                    return; //File doesn't exist
                 }
+
+                _file = new FileInfo(target);
+
+            }
+            else if (_taskType == ProcessTaskType.SQLFile)
+            {
+                if (!BasicActivator.TypeText("Enter a name for the SQL file", "File name", 100, "myscript.sql",
+                        out var selected, false)) return;
+
+                var target = Path.Combine(_loadDirectory.ExecutablesPath.FullName, selected);
+
+                if (!target.EndsWith(".sql"))
+                    target += ".sql";
+
+                //create it if it doesn't exist
+                if (!File.Exists(target))
+                    File.WriteAllText(target, "/*todo Type some SQL*/");
+
+                _file = new FileInfo(target);
+
             }
             else if (_taskType == ProcessTaskType.Executable)
             {
@@ -117,6 +130,7 @@ public override string GetCommandName()
         {
             ProcessTaskType.Executable => "Add Run .exe File Task",
             ProcessTaskType.SQLFile => "Add Run SQL Script Task",
+            ProcessTaskType.SQLBakFile => "Add SQL backup File Task",
             _ => throw new ArgumentOutOfRangeException()
         };
     }
@@ -126,6 +140,7 @@ public override Image<Rgba32> GetImage(IIconProvider iconProvider)
         return _taskType switch
         {
             ProcessTaskType.SQLFile => iconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add),
+            ProcessTaskType.SQLBakFile => iconProvider.GetImage(RDMPConcept.SQL, OverlayKind.Add), //todo maybe better
             ProcessTaskType.Executable => IconOverlayProvider.GetOverlayNoCache(
                 Image.Load<Rgba32>(CatalogueIcons.Exe), OverlayKind.Add),
             _ => null

Rdmp.Core/Curation/Data/DataLoad/ProcessTask.cs

@@ -198,7 +198,9 @@ public void Check(ICheckNotifier notifier)
             case ProcessTaskType.SQLFile:
                 CheckFileExistenceAndUniqueness(notifier);
                 CheckForProblemsInSQLFile(notifier);
-
+                break;
+            case ProcessTaskType.SQLBakFile:
+                CheckFileExistenceAndUniqueness(notifier);
                 break;
             case ProcessTaskType.Attacher:
                 break;
@@ -374,6 +376,7 @@ public static bool IsCompatibleStage(ProcessTaskType type, LoadStage stage)
         {
             ProcessTaskType.Executable => true,
             ProcessTaskType.SQLFile => stage != LoadStage.GetFiles,
+            ProcessTaskType.SQLBakFile => stage != LoadStage.GetFiles,
             ProcessTaskType.Attacher => stage == LoadStage.Mounting,
             ProcessTaskType.DataProvider => true,
             ProcessTaskType.MutilateDataTable => stage != LoadStage.GetFiles,

Rdmp.Core/Curation/Data/DataLoad/ProcessTaskType.cs

@@ -22,6 +22,11 @@ public enum ProcessTaskType
     /// </summary>
     SQLFile,
 
+    /// <summary>
+    /// ProcessTask is to import a SQL backup file directly to the server
+    /// </summary>
+    SQLBakFile,
+
     /// <summary>
     /// ProcessTask is to instantiate the IAttacher class Type specified in Path and hydrate its [DemandsInitialization] properties with values matching
     /// ProcessTaskArguments and run it in the specified load stage in an AttacherRuntimeTask wrapper.

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Runtime/ExecuteSqlBakFileRuntimeTask.cs

@@ -0,0 +1,127 @@
+// Copyright (c) The University of Dundee 2018-2023
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using System;
+using System.Data;
+using System.IO;
+using System.Linq;
+using Microsoft.Data.SqlClient;
+using MongoDB.Driver.Core.Servers;
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.DataLoad.Engine.Job;
+using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
+using Rdmp.Core.DataLoad.Modules.Mutilators;
+using Rdmp.Core.ReusableLibraryCode.Checks;
+using Rdmp.Core.ReusableLibraryCode.Progress;
+
+namespace Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Runtime;
+
+/// <summary>
+/// RuntimeTask that executes a single .bak file specified by the user in a ProcessTask with ProcessTaskType SQLBakFile.
+/// </summary>
+public class ExecuteSqlBakFileRuntimeTask : RuntimeTask
+{
+    public string Filepath;
+    private readonly IProcessTask _task;
+
+    private LoadStage _loadStage;
+
+    public ExecuteSqlBakFileRuntimeTask(IProcessTask task, RuntimeArgumentCollection args) : base(task, args)
+    {
+        _task = task;
+        Filepath = task.Path;
+    }
+
+    public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellationToken)
+    {
+        var db = RuntimeArguments.StageSpecificArguments.DbInfo;
+        _loadStage = RuntimeArguments.StageSpecificArguments.LoadStage;
+
+        if (!Exists())
+            throw new Exception($"The sql bak file {Filepath} does not exist");
+
+        string fileOnlyCommand = $"RESTORE FILELISTONLY FROM DISK = '{Filepath}'";
+        using var fileInfo = new DataTable();
+        using var con = (SqlConnection)db.Server.GetConnection();
+        using (var cmd = new SqlCommand(fileOnlyCommand, con))
+          using (var da = new SqlDataAdapter(cmd))
+            da.Fill(fileInfo);
+        DataRow[] primaryFiles = fileInfo.Select("Type = 'D'");
+        DataRow[] logFiles = fileInfo.Select("Type = 'L'");
+        if (primaryFiles.Length != 1 || logFiles.Length != 1)
+        {
+            //Something has gone wrong
+            return ExitCodeType.Error;
+        }
+
+
+        DataRow primaryFile = primaryFiles[0];
+        DataRow logFile = logFiles[0];
+        string primaryFilePhysicalName = primaryFile["PhysicalName"].ToString();
+        string logFilePhysicalName = logFile["PhysicalName"].ToString();
+
+        if (File.Exists(primaryFilePhysicalName) || File.Exists(logFilePhysicalName))
+        {
+            string timestamp = DateTime.Now.Millisecond.ToString();
+            string primaryFileName = primaryFilePhysicalName.Substring(0, primaryFilePhysicalName.Length - 4);
+            string primaryFileExtention = primaryFilePhysicalName.Substring(primaryFilePhysicalName.Length - 4);
+            primaryFilePhysicalName = $"{primaryFileName}_{timestamp}{primaryFileExtention}";
+            string logFileName = logFilePhysicalName.Substring(0, logFilePhysicalName.Length - 4);
+            string logFileExtention = logFilePhysicalName.Substring(logFilePhysicalName.Length - 4);
+            logFilePhysicalName = $"{logFileName}_{timestamp}{logFileExtention}";
+        }
+
+        string name = db.ToString();
+
+        string restoreCommand = @$"
+        use master;
+        ALTER DATABASE {name} SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
+        RESTORE DATABASE {name}
+        FROM DISK = '{Filepath}'
+        WITH MOVE '{primaryFile["LogicalName"]}' TO '{primaryFilePhysicalName}',
+        MOVE '{logFile["LogicalName"]}' TO '{logFilePhysicalName}' ,  NOUNLOAD,  REPLACE,  STATS = 5;
+        ALTER DATABASE {name} SET MULTI_USER;
+        ";
+
+        job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+            $"Executing script {Filepath} ( against {db})"));
+
+        var executer = new ExecuteSqlInDleStage(job, _loadStage);
+        return executer.Execute(restoreCommand, db);
+    }
+
+
+    public override bool Exists() => File.Exists(Filepath);
+
+    public override void Abort(IDataLoadEventListener postLoadEventListener)
+    {
+    }
+
+    public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
+    {
+    }
+
+    public override void Check(ICheckNotifier notifier)
+    {
+        if (string.IsNullOrWhiteSpace(Filepath))
+        {
+            notifier.OnCheckPerformed(
+                new CheckEventArgs($"ExecuteSqlFileTask {_task} does not have a path specified",
+                    CheckResult.Fail));
+            return;
+        }
+
+        if (!File.Exists(Filepath))
+            notifier.OnCheckPerformed(
+                new CheckEventArgs(
+                    $"File '{Filepath}' does not exist! (the only time this would be legal is if you have an exe or a freaky plugin that creates this file)",
+                    CheckResult.Warning));
+        else
+            notifier.OnCheckPerformed(new CheckEventArgs($"Found File '{Filepath}'",
+                CheckResult.Success));
+    }
+}

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Runtime/RuntimeTaskFactory.cs

@@ -35,6 +35,7 @@ public static RuntimeTask Create(IProcessTask task, IStageArgs stageArgs)
             ProcessTaskType.Attacher => new AttacherRuntimeTask(task, args),
             ProcessTaskType.DataProvider => new DataProviderRuntimeTask(task, args),
             ProcessTaskType.MutilateDataTable => new MutilateDataTablesRuntimeTask(task, args),
+            ProcessTaskType.SQLBakFile => new ExecuteSqlBakFileRuntimeTask(task, args),
             _ => throw new Exception($"Cannot create runtime task: Unknown process task type '{task.ProcessTaskType}'")
         };
     }

Rdmp.Core/Icons/IconProvision/StateBasedIconProviders/ProcessTaskStateBasedIconProvider.cs

@@ -41,6 +41,7 @@ public Image<Rgba32> GetImageIfSupportedObject(object o)
             {
                 ProcessTaskType.Executable => _exe,
                 ProcessTaskType.SQLFile => _sql,
+                ProcessTaskType.SQLBakFile => _sql,
                 ProcessTaskType.Attacher => _attacher,
                 ProcessTaskType.DataProvider => _dataProvider,
                 ProcessTaskType.MutilateDataTable => _mutilateDataTables,

Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsLoadStageNode.cs

@@ -42,6 +42,9 @@ public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, Lo
                 case ".sql":
                     return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.SQLFile,
                         targetStage.LoadMetadata, targetStage.LoadStage, f);
+                case ".bak":
+                    return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.SQLBakFile,
+                        targetStage.LoadMetadata, targetStage.LoadStage, f);
                 case ".exe":
                     return new ExecuteCommandCreateNewFileBasedProcessTask(ItemActivator, ProcessTaskType.Executable,
                         targetStage.LoadMetadata, targetStage.LoadStage, f);

Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsProcessTask.cs

@@ -32,6 +32,9 @@ public override void Activate(ProcessTask processTask)
             case ProcessTaskType.SQLFile:
                 ItemActivator.Activate<SqlProcessTaskUI, ProcessTask>(processTask);
                 break;
+            case ProcessTaskType.SQLBakFile:
+                ItemActivator.Activate<SqlProcessTaskUI, ProcessTask>(processTask);
+                break;
         }
     }