Develop

CHANGELOG.md

@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.1.7] 2020-08-17
+
+### Fixed
+
+- Allow the PACS to send us lossy compressed versions if it wants, otherwise we won't be able to receive anything it has in that format
+
+### Added
+
+- Accept video (MPEG/HEVC) content if the PACS offers it
+- Added new cache source `ProcessBasedCacheSource` that calls out to a remote process
+
 ## [2.1.6] 2020-06-17
 
 ### Fixed

Packages.md

@@ -10,4 +10,4 @@
 | Package | Source Code | Version | License | Purpose | Additional Risk Assessment |
 | ------- | ------------| --------| ------- | ------- | -------------------------- |
 | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
-| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.4](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.4) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
+| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.7](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |

Plugin/netcoreapp2.2/netcoreapp2.2.csproj

@@ -17,12 +17,12 @@
     <PackageReference Include="System.Net.Http" Version="4.*" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Google.Protobuf" Version="3.12.3" />
+    <PackageReference Include="Google.Protobuf" Version="3.13.0" />
     <PackageReference Include="HIC.DicomTypeTranslation" Version="2.3.0" />
     <PackageReference Include="Oracle.ManagedDataAccess.Core" Version="2.19.80" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />
     <PackageReference Include="System.ComponentModel.Composition" Version="4.7.0" />
-    <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" Version="4.3.0" />
   </ItemGroup>
   <ItemGroup>

Rdmp.Dicom.Library.nuspec

@@ -14,7 +14,7 @@
         <copyright>Copyright 2018-2019</copyright>
         <dependencies>
           <dependency id="HIC.DicomTypeTranslation" version="2.3.0" />
-          <dependency id="HIC.RDMP.Plugin" version="4.1.4" />
+          <dependency id="HIC.RDMP.Plugin" version="4.1.7" />
         </dependencies>
     </metadata>
   <files>

Rdmp.Dicom.Tests/Rdmp.Dicom.Tests.csproj

@@ -37,8 +37,8 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="HIC.BadMedicine.Dicom" Version="0.0.6" />
-    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="4.1.5" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
+    <PackageReference Include="HIC.RDMP.Plugin.Test" Version="4.1.7" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.0" />
     <PackageReference Include="NUnit" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.17.0" />
     <PackageReference Include="NunitXml.TestLogger" Version="2.1.62" />

Rdmp.Dicom.Tests/TestProcessBasedCacheSource.cs

@@ -0,0 +1,80 @@
+using MapsDirectlyToDatabaseTable;
+using NUnit.Framework;
+using Rdmp.Core.Caching.Requests;
+using Rdmp.Core.Caching.Requests.FetchRequestProvider;
+using Rdmp.Core.Curation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Curation.Data.Cache;
+using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.Startup;
+using Rdmp.Dicom.Cache.Pipeline;
+using ReusableLibraryCode.Checks;
+using ReusableLibraryCode.Progress;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Tests.Common;
+
+namespace Rdmp.Dicom.Tests
+{
+    class TestProcessBasedCacheSource : UnitTests
+    {
+        [Test]
+        public void TestWithEcho()
+        {
+            var source = new ProcessBasedCacheSource();
+
+            if(IsLinux)
+            {
+                source.Command = "/bin/echo";
+                source.Args = "Hey Thomas go get %s and store in %d";
+            }
+            else
+            {
+                source.Command = "cmd.exe";
+                source.Args = "/c echo Hey Thomas go get %s and store in %d";
+            }
+            source.TimeFormat = "dd/MM/yy";
+            source.ThrowOnNonZeroExitCode = true;
+
+            // What dates to load
+            var cp = WhenIHaveA<CacheProgress>();
+            cp.CacheFillProgress = new DateTime(2001,12,24);
+            cp.SaveToDatabase();
+
+            // Where to put files
+            var lmd = cp.LoadProgress.LoadMetadata;
+
+            var dir = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory);
+            var loadDir = LoadDirectory.CreateDirectoryStructure(dir,"blah",true);
+
+            lmd.LocationOfFlatFiles = loadDir.RootPath.FullName;            
+            lmd.SaveToDatabase();
+
+            source.PreInitialize(new CacheFetchRequestProvider(cp), new ThrowImmediatelyDataLoadEventListener());
+            source.PreInitialize(cp.CatalogueRepository,new ThrowImmediatelyDataLoadEventListener());
+            source.PreInitialize(new PermissionWindow(cp.CatalogueRepository),new ThrowImmediatelyDataLoadEventListener());
+
+            var toMem = new ToMemoryDataLoadEventListener(true);
+            var fork = new ForkDataLoadEventListener(toMem,new ThrowImmediatelyDataLoadEventListener(){WriteToConsole = true});
+
+            source.GetChunk(fork,new GracefulCancellationToken());
+
+            Assert.Contains($"Hey Thomas go get 24/12/01 and store in {Path.Combine(loadDir.Cache.FullName,"ALL")}",toMem.GetAllMessagesByProgressEventType()[ProgressEventType.Information].Select(v=>v.Message).ToArray());
+
+
+
+        }
+
+        public static bool IsLinux
+        {
+            get
+            {
+                int p = (int) Environment.OSVersion.Platform;
+                return (p == 4) || (p == 6) || (p == 128);
+            }
+        }
+    }
+}

Rdmp.Dicom.UI/CreateNewImagingDatasetUI.cs

@@ -51,23 +51,26 @@ private bool CreateDatabaseIfNotExists(DiscoveredDatabase db)
 
         private void btnCreateSuiteWithTemplate_Click(object sender, EventArgs e)
         {
-            OpenFileDialog ofd = new OpenFileDialog();
-            ofd.Filter = "Imaging Template|*.it";
+            string filename;
+            using (OpenFileDialog ofd = new OpenFileDialog()
+            {
+                Filter = "Imaging Template|*.it"
+            })
+            {
+                if (ofd.ShowDialog() != DialogResult.OK)
+                    return;
+                filename = ofd.FileName;
+            }
 
-            if (ofd.ShowDialog() == DialogResult.OK)
+            try
+            {
+                var yaml = File.ReadAllText(filename);
+                var template = ImageTableTemplateCollection.LoadFrom(yaml);
+                CreateSuite(template);
+            }
+            catch (Exception exception)
             {
-                try
-                {
-                    var yaml = File.ReadAllText(ofd.FileName);
-
-                    var template = ImageTableTemplateCollection.LoadFrom(yaml);
-
-                    CreateSuite(template);
-                }
-                catch (Exception exception)
-                {
-                    ExceptionViewer.Show(exception);
-                }
+                ExceptionViewer.Show(exception);
             }
         }
 
@@ -78,17 +81,19 @@ private void CreateSuite(ImageTableTemplateCollection template)
             if (!CreateDatabaseIfNotExists(db))
                 return;
 
-
-            FolderBrowserDialog dialog = new FolderBrowserDialog();
-            DirectoryInfo dir = null;
-            dialog.Description = "Select Project Directory (For Sql scripts/Executables etc)";
 
-            //if we are creating a load we need to know where to store load scripts etc
-            if(cbCreateLoad.Checked)
-                if (dialog.ShowDialog() == DialogResult.OK)
-                    dir = new DirectoryInfo(dialog.SelectedPath);
-                else
-                    return;
+            DirectoryInfo dir = null;
+            using (FolderBrowserDialog dialog = new FolderBrowserDialog() {
+                Description = "Select Project Directory (For Sql scripts/Executables etc)"
+            })
+            {
+                //if we are creating a load we need to know where to store load scripts etc
+                if (cbCreateLoad.Checked)
+                    if (dialog.ShowDialog() == DialogResult.OK)
+                        dir = new DirectoryInfo(dialog.SelectedPath);
+                    else
+                        return;
+            }
 
             var cmd = new ExecuteCommandCreateNewImagingDatasetSuite(_activator.RepositoryLocator, db,dir);
             cmd.DicomSourceType = rbJsonSources.Checked ? typeof(DicomDatasetCollectionSource) : typeof(DicomFileCollectionSource);

Rdmp.Dicom.UI/Rdmp.Dicom.UI.csproj

@@ -12,7 +12,7 @@
     <ProjectReference Include="..\Rdmp.Dicom\Rdmp.Dicom.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="HIC.RDMP.Plugin.UI" Version="4.1.5" />
+    <PackageReference Include="HIC.RDMP.Plugin.UI" Version="4.1.7" />
     <PackageReference Include="System.Drawing.Common" Version="4.7.0" />
   </ItemGroup>
   <ItemGroup>

Rdmp.Dicom.UI/TagElevationXmlUI.cs

@@ -84,7 +84,7 @@ private void RunChecks()
 
             try
             {
-                var collection = new TagElevationRequestCollection(queryEditor.Text);
+                new TagElevationRequestCollection(queryEditor.Text);
                 RagSmiley1.OnCheckPerformed(new CheckEventArgs("Succesfully created elevator",CheckResult.Success));
             }
             catch(Exception ex)

Rdmp.Dicom/Attachers/Routing/AutoRoutingAttacher.cs

@@ -5,7 +5,6 @@
 using System.Linq;
 using System.Text.RegularExpressions;
 using FAnsi.Implementations.MySql;
-using MySql.Data.MySqlClient;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.DataLoad;
 using Rdmp.Core.Curation.Data.Pipelines;
@@ -171,9 +170,6 @@ public IPipelineUseCase GetDesignTimePipelineUseCase(RequiredPropertyInfo proper
         #region Process Results Of Pipeline Read
         public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener listener,GracefulCancellationToken cancellationToken)
         {
-
-            //todo: This really shouldn't be needed surely
-            MySqlConnection.ClearAllPools();
             MySqlBulkCopy.BulkInsertBatchTimeoutInSeconds = int.MaxValue; //forever
 
             _sw.Start();

Rdmp.Dicom/Cache/Pipeline/CachingSCP.cs

@@ -27,6 +27,23 @@ public class CachingSCP : DicomService, IDicomServiceProvider, IDicomCStoreProvi
             DicomTransferSyntax.JPEGProcess14SV1,
             DicomTransferSyntax.JPEGProcess14,
             DicomTransferSyntax.RLELossless,
+
+            // Lossy - if that's all the PACS has, that's all it can give us
+            DicomTransferSyntax.JPEGLSNearLossless,
+            DicomTransferSyntax.JPEG2000Lossy,
+            DicomTransferSyntax.JPEGProcess1,
+            DicomTransferSyntax.JPEGProcess2_4,
+
+            // Also allow video files, just in case
+            DicomTransferSyntax.HEVCH265Main10ProfileLevel51,
+            DicomTransferSyntax.HEVCH265MainProfileLevel51,
+            DicomTransferSyntax.MPEG2,
+            DicomTransferSyntax.MPEG2MainProfileHighLevel,
+            DicomTransferSyntax.MPEG4AVCH264BDCompatibleHighProfileLevel41,
+            DicomTransferSyntax.MPEG4AVCH264HighProfileLevel41,
+            DicomTransferSyntax.MPEG4AVCH264HighProfileLevel42For2DVideo,
+            DicomTransferSyntax.MPEG4AVCH264HighProfileLevel42For3DVideo,
+            DicomTransferSyntax.MPEG4AVCH264StereoHighProfileLevel42,
 
             // Uncompressed
             DicomTransferSyntax.ExplicitVRLittleEndian,

Rdmp.Dicom/Cache/Pipeline/Dicom/DicomRequestSender.cs

@@ -70,7 +70,7 @@ public void ThrottleRequest(DicomClient client, CancellationToken cancellationTo
             SendRequest(client,cancellationToken);
             transferTimer.Stop();
             //valuein mills
-            var delay = ((int)(_dicomConfiguration.RequestDelayFactor * (1000 * transferTimer.Elapsed.Seconds)) + _dicomConfiguration.RequestCooldownInMilliseconds);
+            var delay = ((int)(_dicomConfiguration.RequestDelayFactor * (1000f * transferTimer.Elapsed.Seconds)) + _dicomConfiguration.RequestCooldownInMilliseconds);
             if (delay > 0)
             {
                 _listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Requests sleeping for " + delay / 1000 + "seconds"));

Rdmp.Dicom/Cache/Pipeline/Ordering/HierarchyBasedOrder.cs

@@ -411,10 +411,7 @@ public int Total()
                 {
                     foreach (var series in study.Series.Values)
                     {
-                        foreach (var image in series.Images.Values)
-                        {
-                            count++;
-                        }
+                        count += series.Images.Values.Count;
                     }
                 }
             }

Rdmp.Dicom/Cache/Pipeline/ProcessBasedCacheSource.cs

@@ -0,0 +1,106 @@
+using Rdmp.Core.Caching.Pipeline.Sources;
+using Rdmp.Core.Caching.Requests;
+using Rdmp.Core.Curation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.DataFlowPipeline;
+using ReusableLibraryCode.Checks;
+using ReusableLibraryCode.Progress;
+using System;
+using System.Diagnostics;
+
+namespace Rdmp.Dicom.Cache.Pipeline
+{
+    public class ProcessBasedCacheSource : CacheSource<SMIDataChunk>
+    {
+        [DemandsInitialization(@"Process to start (path only)",Mandatory = true)]
+        public string Command {get;set;}
+
+        [DemandsInitialization(@"Arguments to provide to the Process.  Template with 
+%s start time
+%e end time time to fetch
+%d directory to put files fetched
+Example:. './GetImages.exe ""%s"" ""%e%""'")]
+        public string Args {get;set;}
+
+        [DemandsInitialization("The datetime format for %s and %e.",Mandatory = true,DefaultValue = "yyyy-MM-dd HH:mm:ss")]
+        public string TimeFormat {get;set;}
+
+        [DemandsInitialization("True to throw an Exception if the process run returns a nonzero exit code", DefaultValue = true)]
+        public bool ThrowOnNonZeroExitCode {get;set;}
+
+        public override void Abort(IDataLoadEventListener listener)
+        {
+
+        }
+
+        public override void Check(ICheckNotifier notifier)
+        {
+
+        }
+
+        public override void Dispose(IDataLoadEventListener listener, Exception pipelineFailureExceptionIfAny)
+        {
+
+        }
+
+        public override SMIDataChunk DoGetChunk(ICacheFetchRequest request, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken)
+        {                     
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,$"ProcessBasedCacheSource version is {typeof(ProcessBasedCacheSource).Assembly.GetName().Version}.  Assembly is {typeof(ProcessBasedCacheSource).Assembly} " ));
+
+            // Where we are putting the files
+            var cacheDir = new LoadDirectory(Request.CacheProgress.LoadProgress.LoadMetadata.LocationOfFlatFiles).Cache;
+            var cacheLayout = new SMICacheLayout(cacheDir, new SMICachePathResolver("ALL"));
+
+            Chunk = new SMIDataChunk(Request)
+            {
+                FetchDate = Request.Start,
+                Modality = "ALL",
+                Layout = cacheLayout
+            };
+
+            var workingDirectory = cacheLayout.GetLoadCacheDirectory(listener);
+
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Working directory is:" + workingDirectory));
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Fetch Start is:" + request.Start));
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Fetch End is:" + request.End));
+
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Command is:" + Command));
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Args template is:" + Args));
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Datetime format is:" + TimeFormat));
+
+
+            string args = Args
+                .Replace("%s",request.Start.ToString(TimeFormat))
+                .Replace("%e",request.End.ToString(TimeFormat))
+                .Replace("%d",workingDirectory.FullName);
+
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,"Args resolved is:" + args));
+
+            using(var p = new Process())
+            {
+                p.StartInfo.FileName = Command;
+                p.StartInfo.Arguments = args;
+                p.StartInfo.UseShellExecute = false;
+                p.StartInfo.RedirectStandardOutput = true;
+                p.OutputDataReceived += (sender, a) => listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,a.Data));
+
+                p.Start();
+                p.BeginOutputReadLine();
+
+                p.WaitForExit();
+
+                listener.OnNotify(this,new NotifyEventArgs( p.ExitCode == 0 ? ProgressEventType.Information : ProgressEventType.Warning , "Process exited with code " + p.ExitCode));
+
+                if(p.ExitCode != 0 && ThrowOnNonZeroExitCode)
+                    throw new Exception("Process exited with code " + p.ExitCode);
+            }
+
+            return Chunk;
+        }
+
+        public override SMIDataChunk TryGetPreview()
+        {
+            return null;
+        }
+    }
+}