Task/rdmp 202 improved load metadata api

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.2.1] - Unreleased
+
+## Changed
+- Update to LoadDirectory API to allow for population of LoadMetadatas
 ## [8.2.0] - Unreleased
 
 ## Changed

Rdmp.Core.Tests/Caching/Integration/CachingHostTests.cs

@@ -35,15 +35,10 @@
         if (testDir.Exists)
             Directory.Delete(testDir.FullName, true);
 
-        var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "Test");
-
-
         var cp = WhenIHaveA<CacheProgress>();
         var loadMetadata = cp.LoadProgress.LoadMetadata;
-        loadMetadata.LocationOfForLoadingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForLoadingPath);
-        loadMetadata.LocationOfForArchivingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForArchivingPath);
-        loadMetadata.LocationOfExecutablesDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultExecutablesPath);
-        loadMetadata.LocationOfCacheDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultCachePath);
+
+        var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "Test", false, (LoadMetadata)loadMetadata);
 
         // This feels a bit nasty, but quick and much better than having the test wait for an arbitrary time period.
         var listener = new ExpectedNotificationListener("Download not permitted at this time, sleeping for 60 seconds");

Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs

@@ -55,16 +55,11 @@ public void FetchMultipleDays_Success(bool singleDay)
         pipeline.Destination.Returns(destinationComponent);
         pipeline.Repository.Returns(CatalogueRepository);
         pipeline.PipelineComponents.Returns(Enumerable.Empty<IPipelineComponent>().OrderBy(o => o).ToList());
+        var lmd = Substitute.For<ILoadMetadata>();
 
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme",
-                true);
-
-        var lmd = Substitute.For<ILoadMetadata>();
-        lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForLoading"));
-        lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForArchiving"));
-        lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName ,"Executables");
-        lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "Cache"));
+                true, lmd);
 
         var loadProgress = Substitute.For<ILoadProgress>();
         loadProgress.OriginDate.Returns(new DateTime(2001, 01, 01));

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndCacheTest.cs

@@ -48,11 +48,7 @@ protected override void SetUp()
         _lmd = new LoadMetadata(CatalogueRepository, "Ive got a lovely bunch o' coconuts");
         _LoadDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                @"EndToEndCacheTest", true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultCachePath);
+                @"EndToEndCacheTest", true,_lmd);
         _lmd.SaveToDatabase();
 
         Clear(_LoadDirectory);

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndDLETest.cs

@@ -46,10 +46,7 @@ public void TestDle_DodgyColumnNames(DatabaseType dbType)
 
         var cata = Import(tbl);
         var lmd = new LoadMetadata(CatalogueRepository, nameof(TestDle_DodgyColumnNames));
-        lmd.LocationOfForLoadingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultCachePath);
+        LoadDirectory.PopulateLoadMetadata(lmd);
         lmd.SaveToDatabase();
 
         CreateFlatFileAttacher(lmd, "Troll.csv", cata.GetTableInfoList(false).Single());

Rdmp.Core.Tests/Curation/Integration/LoadProgressUnitTests.cs

@@ -65,15 +65,10 @@ public void LoadProgress_JobFactory_NoDates()
         var stratFactory =
             new JobDateGenerationStrategyFactory(new AnyAvailableLoadProgressSelectionStrategy(lp.LoadMetadata));
         var strat = stratFactory.Create(lp, ThrowImmediatelyDataLoadEventListener.Quiet);
+        var lmd = lp.LoadMetadata;
 
         var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory),
-            "LoadProgress_JobFactory_NoDates", true);
-
-        var lmd = lp.LoadMetadata;
-        lmd.LocationOfForLoadingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultCachePath);
+            "LoadProgress_JobFactory_NoDates", true, (LoadMetadata)lmd);
 
         foreach (var cata in lmd.GetAllCatalogues())
         {

Rdmp.Core.Tests/DataLoad/Engine/Integration/CheckingTests/ProcessTaskCheckingTests.cs

@@ -35,11 +35,7 @@
         _dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ProcessTaskCheckingTests"));
         _dir.Create();
 
-        var hicdir = LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultCachePath);
+        var hicdir = LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true, _lmd);
         _lmd.SaveToDatabase();
 
         var c = new Catalogue(CatalogueRepository, "c");
@@ -128,13 +124,9 @@
     {
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "DelMeProjDir", true);
+                "DelMeProjDir", true, _lmd);
         try
         {
-            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;
@@ -158,13 +150,9 @@
     {
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "DelMeProjDir", true);
+                "DelMeProjDir", true, _lmd);
         try
         {
-            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;

Rdmp.Core.Tests/DataLoad/Engine/Integration/CrossDatabaseTypeTests/CrossDatabaseDataLoadTests.cs

@@ -155,7 +155,6 @@ public void Load(DatabaseType databaseType, TestCase testCase)
 
         //define a new load configuration
         var lmd = new LoadMetadata(CatalogueRepository, "MyLoad");
-
         if (testCase == TestCase.NoTrigger)
         {
             lmd.IgnoreTrigger = true;

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadEngineTestsBase.cs

@@ -71,11 +71,8 @@ protected static LoadDirectory SetupLoadDirectory(LoadMetadata lmd)
     {
         var projectDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "MyLoadDir", true);
-        lmd.LocationOfForLoadingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultCachePath);
+                "MyLoadDir", true,lmd);
+
         lmd.SaveToDatabase();
 
         return projectDirectory;

Rdmp.Core.Tests/DataLoad/Engine/Integration/HICPipelineTests.cs

@@ -27,6 +27,7 @@
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.DataAccess;
 using Rdmp.Core.ReusableLibraryCode.Progress;
+using Terminal.Gui;
 using Tests.Common;
 
 namespace Rdmp.Core.Tests.DataLoad.Engine.Integration;

Rdmp.Core.Tests/DataLoad/Engine/Integration/JobDateGenerationStrategyFactoryTestsIntegration.cs

@@ -126,11 +126,7 @@ public void CacheProvider_NoPipeline()
 
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme",
-                true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
+                true, _lmd);
         _lmd.SaveToDatabase();
         try
         {
@@ -156,11 +152,7 @@ public void CacheProvider_NoCacheProgress()
 
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme",
-                true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
+                true, _lmd);
         _lmd.SaveToDatabase();
 
         var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);
@@ -197,11 +189,7 @@ public void CacheProvider_Normal()
 
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme",
-                true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
+                true, _lmd);
         _lmd.SaveToDatabase();
 
         var pipeAssembler = new TestDataPipelineAssembler("CacheProvider_Normal", CatalogueRepository);

Rdmp.Core.Tests/DataLoad/Engine/Integration/PayloadTest.cs

@@ -39,12 +39,8 @@ public void TestPayloadInjection()
 
         var lmd = new LoadMetadata(CatalogueRepository, "Loading");
         var filePath = LoadDirectory
-                .CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true)
+                .CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme", true,lmd)
                 .RootPath.FullName;
-        lmd.LocationOfForLoadingDirectory = Path.Combine(filePath , lmd.DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(filePath , lmd.DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(filePath , lmd.DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(filePath , lmd.DefaultCachePath);
         lmd.SaveToDatabase();
 
         MEF.AddTypeToCatalogForTesting(typeof(TestPayloadAttacher));

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewDataLoadDirectory.cs

@@ -61,16 +61,12 @@ public override void Execute()
 
         var loadDir =
             string.IsNullOrWhiteSpace(newFolderName)
-                ? LoadDirectory.CreateDirectoryStructure(d.Parent, d.Name, true)
-                : LoadDirectory.CreateDirectoryStructure(d, newFolderName, true);
+                ? LoadDirectory.CreateDirectoryStructure(d.Parent, d.Name, true, LoadMetadata)
+                : LoadDirectory.CreateDirectoryStructure(d, newFolderName, true, LoadMetadata);
 
         // if we have a load then update the path to this location we just created
         if (LoadMetadata != null)
         {
-            LoadMetadata.LocationOfForLoadingDirectory = Path.Combine(loadDir.RootPath.FullName ,LoadMetadata.DefaultForLoadingPath);
-            LoadMetadata.LocationOfForArchivingDirectory = Path.Combine(loadDir.RootPath.FullName, LoadMetadata.DefaultForArchivingPath);
-            LoadMetadata.LocationOfExecutablesDirectory = Path.Combine(loadDir.RootPath.FullName , LoadMetadata.DefaultExecutablesPath);
-            LoadMetadata.LocationOfCacheDirectory = Path.Combine(loadDir.RootPath.FullName , LoadMetadata.DefaultCachePath);
             LoadMetadata.SaveToDatabase();
             Publish(LoadMetadata);
         }

Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCreateNewSplitDataLoadDirectory.cs

@@ -114,11 +114,7 @@ public override void Execute()
         // if we have a load then update the path to this location we just created
         if (LoadMetadata != null)
         {
-            LoadMetadata.LocationOfForLoadingDirectory = loadDir.ForLoading.FullName;
-            LoadMetadata.LocationOfForArchivingDirectory = loadDir.ForArchiving.FullName;
-            LoadMetadata.LocationOfExecutablesDirectory = loadDir.ExecutablesPath.FullName;
-            LoadMetadata.LocationOfCacheDirectory = loadDir.Cache.FullName;
-            LoadMetadata.SaveToDatabase();
+            loadDir.PopulateLoadMetadata(LoadMetadata);
             Publish(LoadMetadata);
         }
     }

Rdmp.Core/Curation/LoadDirectory.cs

@@ -4,6 +4,7 @@
 // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
+using Rdmp.Core.Curation.Data.DataLoad;
 using System;
 using System.IO;
 using System.Linq;
@@ -52,7 +53,7 @@ public class LoadDirectory : ILoadDirectory
     /// </summary>
     /// <param name="rootPath"></param>
     /// <param name="validate"></param>
-    public LoadDirectory(string rootPath, bool validate=true)
+    public LoadDirectory(string rootPath, bool validate = true)
     {
         if (string.IsNullOrWhiteSpace(rootPath))
             throw new Exception("Root path was blank, there is no LoadDirectory path specified?");
@@ -81,7 +82,7 @@ public LoadDirectory(string ForLoadingPath, string ForArchivingPath, string Exec
             throw new Exception($"One if the LoadDirectory Paths was blank. ForLoading: {ForLoading}. ForArchiving: {ForArchivingPath}. Cache: {CachePath}. Extractables:{ExecutablesPath}");
         ForLoading = new DirectoryInfo(ForLoadingPath);
         ForArchiving = new DirectoryInfo(ForArchivingPath);
-        ExecutablesPath =new DirectoryInfo(ExecutablesPathString);
+        ExecutablesPath = new DirectoryInfo(ExecutablesPathString);
         Cache = new DirectoryInfo(CachePath);
     }
 
@@ -102,9 +103,10 @@ private DirectoryInfo FindFolderInPathOrThrow(DirectoryInfo path, string folderN
     /// <param name="parentDir">Parent folder to create the tree in e.g. c:\temp</param>
     /// <param name="dirName">Root folder name for the DLE e.g. LoadingBiochem</param>
     /// <param name="overrideExistsCheck">Determines behaviour if the folder already exists and contains files.  True to carry on, False to throw an Exception</param>
+    /// <param name="loadMetadataToPopulate">Optional loadMetadata object to populate with the created locations</param>
     /// <returns></returns>
     public static LoadDirectory CreateDirectoryStructure(DirectoryInfo parentDir, string dirName,
-        bool overrideExistsCheck = false)
+        bool overrideExistsCheck = false, ILoadMetadata loadMetadataToPopulate = null)
     {
         if (!parentDir.Exists)
             parentDir.Create();
@@ -128,7 +130,23 @@ public static LoadDirectory CreateDirectoryStructure(DirectoryInfo parentDir, st
         swExampleFixedWidth.Close();
 
         projectDir.CreateSubdirectory("Executables");
+        if (loadMetadataToPopulate != null)
+        {
+            loadMetadataToPopulate.LocationOfForLoadingDirectory = Path.Combine(projectDir.FullName, "Data", "ForLoading");
+            loadMetadataToPopulate.LocationOfForArchivingDirectory = Path.Combine(projectDir.FullName, "Data", "ForArchiving");
+            loadMetadataToPopulate.LocationOfExecutablesDirectory = Path.Combine(projectDir.FullName, "Executables");
+            loadMetadataToPopulate.LocationOfCacheDirectory = Path.Combine(projectDir.FullName, "Data", "Cache");
+        }
 
         return new LoadDirectory(projectDir.FullName);
     }
+
+    public void PopulateLoadMetadata(ILoadMetadata loadMetadata)
+    {
+        loadMetadata.LocationOfForLoadingDirectory = ForLoading.FullName;
+        loadMetadata.LocationOfForArchivingDirectory = ForArchiving.FullName;
+        loadMetadata.LocationOfExecutablesDirectory = ExecutablesPath.FullName;
+        loadMetadata.LocationOfCacheDirectory = Cache.FullName;
+
+    }
 }