marge develop into main

.github/workflows/build.yml

@@ -2,7 +2,10 @@ name: Build
 
 # Run this workflow every time a new commit pushed to your repository
 
-on: push
+on: 
+  push:
+  schedule:
+    - cron: '0 1 * * *'
 
 env:
   DOTNET_NOLOGO: 1
@@ -72,7 +75,6 @@ jobs:
           files: ./db-ui.lcov ./db-core.lcov
           flag-name: unit tests
 
-
   tests_file_system:
     name: Run File System Tests
     runs-on: windows-latest
@@ -145,7 +147,7 @@ jobs:
         shell: bash
         run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT
       - name: Setup .NET Core
-        uses: actions/setup-dotnet@v4.0.0
+        uses: actions/setup-dotnet@v4.0.1
         with:
           dotnet-version: 7.0.x
       - name: BundleSource

.github/workflows/docker.yml

@@ -17,7 +17,7 @@ jobs:
         with:
           submodules: true
       - name: Setup .NET Core
-        uses: actions/setup-dotnet@v4.0.0
+        uses: actions/setup-dotnet@v4.0.1
         with:
           dotnet-version: 6.0.x
       - name: Cache Nuget

Application/ResearchDataManagementPlatform/Menus/RDMPTopMenuStripUI.cs

@@ -74,6 +74,7 @@ public partial class RDMPTopMenuStripUI : RDMPUserControl
 
     private SaveMenuItem _saveToolStripMenuItem;
     private AtomicCommandUIFactory _atomicCommandUIFactory;
+    private ConnectionStringsYamlFile _connectionStringsFileInUse;
 
     public RDMPTopMenuStripUI()
     {
@@ -87,10 +88,10 @@ private void BuildSwitchInstanceMenuItems()
         // somehow app was launched without populating the load args
         if (args == null) return;
 
-        var origYamlFile = args.ConnectionStringsFileLoaded;
+        _connectionStringsFileInUse = args.ConnectionStringsFileLoaded;
 
         //default settings were used if no yaml file was specified or the file specified did not exist
-        var defaultsUsed = origYamlFile == null;
+        var defaultsUsed = _connectionStringsFileInUse == null;
 
         // if defaults were not used then it is valid to switch to them
         switchToDefaultSettings.Enabled = !defaultsUsed;
@@ -100,11 +101,11 @@ private void BuildSwitchInstanceMenuItems()
 
         // load the yaml files in the RDMP binary directory
         var exeDir = UsefulStuff.GetExecutableDirectory();
-        AddMenuItemsForSwitchingToInstancesInYamlFilesOf(origYamlFile, exeDir);
+        AddMenuItemsForSwitchingToInstancesInYamlFilesOf(_connectionStringsFileInUse, exeDir);
 
         // also add yaml files from wherever they got their original yaml file
-        if (origYamlFile?.FileLoaded != null && !exeDir.FullName.Equals(origYamlFile.FileLoaded.Directory?.FullName))
-            AddMenuItemsForSwitchingToInstancesInYamlFilesOf(origYamlFile, origYamlFile.FileLoaded.Directory);
+        if (_connectionStringsFileInUse?.FileLoaded != null && !exeDir.FullName.Equals(_connectionStringsFileInUse.FileLoaded.Directory?.FullName))
+            AddMenuItemsForSwitchingToInstancesInYamlFilesOf(_connectionStringsFileInUse, _connectionStringsFileInUse.FileLoaded.Directory);
     }
 
     private void AddMenuItemsForSwitchingToInstancesInYamlFilesOf(ConnectionStringsYamlFile origYamlFile,
@@ -162,6 +163,16 @@ private void configureExternalServersToolStripMenuItem_Click(object sender, Even
         new ExecuteCommandConfigureDefaultServers(Activator).Execute();
     }
 
+    private void launchDefaultInstance(object sender, EventArgs e)
+    {
+        if (_connectionStringsFileInUse != null)
+        {
+            var exeName = Path.Combine(UsefulStuff.GetExecutableDirectory().FullName,
+            Process.GetCurrentProcess().ProcessName);
+            Process.Start(exeName);
+        }
+    }
+
     private void setTicketingSystemToolStripMenuItem_Click(object sender, EventArgs e)
     {
         var ui = new TicketingSystemConfigurationUI();
@@ -291,7 +302,7 @@ public void SetWindowManager(WindowManager windowManager)
         // Location menu
         instancesToolStripMenuItem.DropDownItems.Add(_atomicCommandUIFactory.CreateMenuItem(
             new ExecuteCommandChoosePlatformDatabase(Activator.RepositoryLocator)
-                { OverrideCommandName = "Change Default Instance" }));
+            { OverrideCommandName = "Change Default Instance" }));
 
         Activator.Theme.ApplyTo(menuStrip1);
 
@@ -463,7 +474,7 @@ private void navigateForwardToolStripMenuItem_Click(object sender, EventArgs e)
         _windowManager.Navigation.Forward(true);
     }
 
-    
+
 
     private void checkForUpdatesToolStripMenuItem_Click(object sender, EventArgs e)
     {

Application/ResearchDataManagementPlatform/Program.cs

@@ -7,7 +7,6 @@
 using System;
 using System.Runtime.InteropServices;
 using CommandLine;
-using Rdmp.Core.Logging.PastEvents;
 using Rdmp.Core.ReusableLibraryCode;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Settings;

CHANGELOG.md

@@ -1,9 +1,19 @@
+
 # Changelog
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.2.1] - 2024-07-18
+
+- Add Release status options to the ticketing system
+- Improve Interface for Lookup table generation
+- Add directory validity checking to data loads
+- Open plugin files read-only to avoid permissions errors on Linux
+- Improve PK mapping for ExtractionIdentifiers when extracting data
+- Fix issue with default instance button not launching instance
+
 ## [8.2.0] - 2024-07-09
 
 ## Changed

Directory.Packages.props

@@ -5,7 +5,7 @@
 		<PackageVersion Include="ExcelNumberFormat" Version="1.1.0"/>
 		<PackageVersion Include="FluentFTP" Version="50.1.0"/>
 		<PackageVersion Include="HIC.SynthEHR" Version="2.0.0"/>
-		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.5"/>
+		<PackageVersion Include="HIC.FAnsiSql" Version="3.2.6"/>
 		<PackageVersion Include="LibArchive.Net" Version="0.1.5"/>
 		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
 			<PrivateAssets>all</PrivateAssets>
@@ -19,15 +19,15 @@
 		<PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="2.1.3"/>
 		<PackageVersion Include="Spectre.Console" Version="0.49.1"/>
 		<PackageVersion Include="SSH.NET" Version="2024.1.0"/>
-		<PackageVersion Include="Terminal.Gui" Version="1.17.0"/>
-		<PackageVersion Include="YamlDotNet" Version="15.3.0"/>
+		<PackageVersion Include="Terminal.Gui" Version="1.17.1"/>
+		<PackageVersion Include="YamlDotNet" Version="16.0.0"/>
 		<PackageVersion Include="ConsoleControl" Version="1.3.0"/>
 		<PackageVersion Include="Autoupdater.NET.Official" Version="1.9.1"/>
 		<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.0"/>
 		<PackageVersion Include="System.Threading.ThreadPool" Version="4.3.0"/>
 		<PackageVersion Include="WeCantSpell.Hunspell" Version="5.0.0"/>
 		<PackageVersion Include="NUnit" Version="4.1.0"/>
-		<PackageVersion Include="ObjectListView.Repack.NET6Plus" Version="2.9.4"/>
+		<PackageVersion Include="ObjectListView.Repack.NET6Plus" Version="2.9.5"/>
 		<PackageVersion Include="Scintilla.NET" Version="5.3.2.9"/>
 		<PackageVersion Include="System.Resources.Extensions" Version="8.0.0"/>
 		<PackageVersion Include="VPKSoft.ScintillaLexers.NET" Version="1.1.16"/>

Rdmp.Core.Tests/Caching/Integration/CachingHostTests.cs

@@ -35,15 +35,10 @@ public void CacheHostOutwithPermissionWindow()
         if (testDir.Exists)
             Directory.Delete(testDir.FullName, true);
 
-        var loadDirectory = LoadDirectory.CreateDirectoryStructure(testDir, "Test");
-
-
         var cp = WhenIHaveA<CacheProgress>();
         var loadMetadata = cp.LoadProgress.LoadMetadata;
-        loadMetadata.LocationOfForLoadingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForLoadingPath);
-        loadMetadata.LocationOfForArchivingDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultForArchivingPath);
-        loadMetadata.LocationOfExecutablesDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultExecutablesPath);
-        loadMetadata.LocationOfCacheDirectory = Path.Combine(loadDirectory.RootPath.FullName , ((LoadMetadata)loadMetadata).DefaultCachePath);
+
+        LoadDirectory.CreateDirectoryStructure(testDir, "Test", false, (LoadMetadata)loadMetadata);
 
         // This feels a bit nasty, but quick and much better than having the test wait for an arbitrary time period.
         var listener = new ExpectedNotificationListener("Download not permitted at this time, sleeping for 60 seconds");

Rdmp.Core.Tests/Caching/Integration/CustomDateCachingTests.cs

@@ -55,16 +55,11 @@ public void FetchMultipleDays_Success(bool singleDay)
         pipeline.Destination.Returns(destinationComponent);
         pipeline.Repository.Returns(CatalogueRepository);
         pipeline.PipelineComponents.Returns(Enumerable.Empty<IPipelineComponent>().OrderBy(o => o).ToList());
+        var lmd = Substitute.For<ILoadMetadata>();
 
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory), "delme",
-                true);
-
-        var lmd = Substitute.For<ILoadMetadata>();
-        lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForLoading"));
-        lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "ForArchiving"));
-        lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName ,"Executables");
-        lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName , Path.Combine("Data", "Cache"));
+                true, lmd);
 
         var loadProgress = Substitute.For<ILoadProgress>();
         loadProgress.OriginDate.Returns(new DateTime(2001, 01, 01));

Rdmp.Core.Tests/CommandExecution/ExecuteCommandCreateDatasetTests.cs

@@ -1,7 +1,6 @@
 
 using NUnit.Framework;
 using Rdmp.Core.CommandExecution.AtomicCommands;
-using System;
 using System.Linq;
 using Rdmp.Core.CommandExecution;
 

Rdmp.Core.Tests/CommandExecution/ExecuteCommandLinkCatalogueToDatasetTests.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Linq;
+using System.Linq;
 using NUnit.Framework;
 using Rdmp.Core.CommandExecution;
 using Rdmp.Core.CommandExecution.AtomicCommands;

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndCacheTest.cs

@@ -48,11 +48,7 @@ protected override void SetUp()
         _lmd = new LoadMetadata(CatalogueRepository, "Ive got a lovely bunch o' coconuts");
         _LoadDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                @"EndToEndCacheTest", true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(_LoadDirectory.RootPath.FullName , _lmd.DefaultCachePath);
+                @"EndToEndCacheTest", true,_lmd);
         _lmd.SaveToDatabase();
 
         Clear(_LoadDirectory);

Rdmp.Core.Tests/CommandLine/AutomationLoopTests/EndToEndDLETest.cs

@@ -13,7 +13,6 @@
 using Tests.Common;
 using Tests.Common.Scenarios;
 using TypeGuesser;
-using System.IO;
 
 namespace Rdmp.Core.Tests.CommandLine.AutomationLoopTests;
 
@@ -46,10 +45,7 @@ public void TestDle_DodgyColumnNames(DatabaseType dbType)
 
         var cata = Import(tbl);
         var lmd = new LoadMetadata(CatalogueRepository, nameof(TestDle_DodgyColumnNames));
-        lmd.LocationOfForLoadingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(LoadDirectory.RootPath.FullName , lmd.DefaultCachePath);
+        LoadDirectory.PopulateLoadMetadata(lmd);
         lmd.SaveToDatabase();
 
         CreateFlatFileAttacher(lmd, "Troll.csv", cata.GetTableInfoList(false).Single());

Rdmp.Core.Tests/Curation/Integration/LoadMetadataTests.cs

@@ -7,7 +7,6 @@
 using System;
 using System.IO;
 using NUnit.Framework;
-using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.DataLoad.Engine.Checks.Checkers;
 using Rdmp.Core.DataLoad.Engine.DatabaseManagement.EntityNaming;

Rdmp.Core.Tests/Curation/Integration/LoadProgressUnitTests.cs

@@ -65,15 +65,10 @@ public void LoadProgress_JobFactory_NoDates()
         var stratFactory =
             new JobDateGenerationStrategyFactory(new AnyAvailableLoadProgressSelectionStrategy(lp.LoadMetadata));
         var strat = stratFactory.Create(lp, ThrowImmediatelyDataLoadEventListener.Quiet);
+        var lmd = lp.LoadMetadata;
 
         var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory),
-            "LoadProgress_JobFactory_NoDates", true);
-
-        var lmd = lp.LoadMetadata;
-        lmd.LocationOfForLoadingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(dir.RootPath.FullName , ((LoadMetadata)lmd).DefaultCachePath);
+            "LoadProgress_JobFactory_NoDates", true, (LoadMetadata)lmd);
 
         foreach (var cata in lmd.GetAllCatalogues())
         {

Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs

@@ -86,6 +86,7 @@ public void ReExtractToADatabaseWithNoNewData()
         var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First();
         var ei = chiColumnInfo.ExtractionInformation;
         ei.IsExtractionIdentifier = true;
+        ei.IsPrimaryKey = true;
         ei.SaveToDatabase();
         var project = new Project(DataExportRepository, "MyProject")
         {
@@ -312,6 +313,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs()
         var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First();
         var ei = chiColumnInfo.ExtractionInformation;
         ei.IsExtractionIdentifier = true;
+        ei.IsPrimaryKey = true;
         ei.SaveToDatabase();
         var project = new Project(DataExportRepository, "MyProject")
         {
@@ -568,6 +570,7 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK()
         var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First();
         var ei = chiColumnInfo.ExtractionInformation;
         ei.IsExtractionIdentifier = true;
+        ei.IsPrimaryKey = true;
         ei.SaveToDatabase();
 
         var surnameInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "surname").First();

Rdmp.Core.Tests/DataLoad/Engine/Integration/CheckingTests/ProcessTaskCheckingTests.cs

@@ -35,11 +35,7 @@ public void CreateTask()
         _dir = new DirectoryInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, "ProcessTaskCheckingTests"));
         _dir.Create();
 
-        var hicdir = LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true);
-        _lmd.LocationOfForLoadingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-        _lmd.LocationOfForArchivingDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-        _lmd.LocationOfExecutablesDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-        _lmd.LocationOfCacheDirectory = Path.Combine(hicdir.RootPath.FullName, _lmd.DefaultCachePath);
+        LoadDirectory.CreateDirectoryStructure(_dir, "ProjDir", true, _lmd);
         _lmd.SaveToDatabase();
 
         var c = new Catalogue(CatalogueRepository, "c");
@@ -128,13 +124,9 @@ public void MEFCompatibleType_NoArgs()
     {
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "DelMeProjDir", true);
+                "DelMeProjDir", true, _lmd);
         try
         {
-            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;
@@ -158,13 +150,9 @@ public void MEFCompatibleType_Passes()
     {
         var projDir =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "DelMeProjDir", true);
+                "DelMeProjDir", true, _lmd);
         try
         {
-            _lmd.LocationOfForLoadingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForLoadingPath);
-            _lmd.LocationOfForArchivingDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultForArchivingPath);
-            _lmd.LocationOfExecutablesDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultExecutablesPath);
-            _lmd.LocationOfCacheDirectory = Path.Combine(projDir.RootPath.FullName, _lmd.DefaultCachePath);
             _task.ProcessTaskType = ProcessTaskType.Attacher;
             _task.LoadStage = LoadStage.Mounting;
             _task.Path = typeof(AnySeparatorFileAttacher).FullName;

Rdmp.Core.Tests/DataLoad/Engine/Integration/CrossDatabaseTypeTests/CrossDatabaseDataLoadTests.cs

@@ -155,7 +155,6 @@ public void Load(DatabaseType databaseType, TestCase testCase)
 
         //define a new load configuration
         var lmd = new LoadMetadata(CatalogueRepository, "MyLoad");
-
         if (testCase == TestCase.NoTrigger)
         {
             lmd.IgnoreTrigger = true;

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataLoadEngineTestsBase.cs

@@ -71,11 +71,8 @@ protected static LoadDirectory SetupLoadDirectory(LoadMetadata lmd)
     {
         var projectDirectory =
             LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.TestDirectory),
-                "MyLoadDir", true);
-        lmd.LocationOfForLoadingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForLoadingPath);
-        lmd.LocationOfForArchivingDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultForArchivingPath);
-        lmd.LocationOfExecutablesDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultExecutablesPath);
-        lmd.LocationOfCacheDirectory = Path.Combine(projectDirectory.RootPath.FullName, lmd.DefaultCachePath);
+                "MyLoadDir", true,lmd);
+
         lmd.SaveToDatabase();
 
         return projectDirectory;

Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs

@@ -13,7 +13,6 @@
 using FAnsi.Discovery;
 using FAnsi.Discovery.TableCreation;
 using NUnit.Framework;
-using NUnit.Framework.Internal;
 using Rdmp.Core.DataFlowPipeline;
 using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations;
 using Rdmp.Core.DataLoad.Triggers;