From 563add8a66d03acc21167d53a0e88b870c330804 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 29 May 2020 09:30:49 +0100 Subject: [PATCH 001/138] Improved the symbols rule factory to generate only the offending words of the input string not the full ProblemValue --- CHANGELOG.md | 1 + .../Out/SymbolsRulesFactory.cs | 61 ++++++++++++++++- .../ReviewerTests/SymbolsRulesFactoryTests.cs | 65 ++++++++++++++++--- 3 files changed, 116 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a154ff960..070c268a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump fo-dicom.Drawing from 4.0.4 to 4.0.5 - Bump HIC.BadMedicine.Dicom from 0.0.5 to 0.0.6 - Pinned fo-dicom.NetCore to 4.0.5 +- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` ## [1.8.1] - 2020-04-17 diff --git a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs index 440e9ad9c..132e81b78 100644 --- a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs @@ -1,4 +1,6 @@ -using System.Text; +using System; +using System.Linq; +using System.Text; using System.Text.RegularExpressions; using Microservices.IsIdentifiable.Reporting; @@ -6,10 +8,67 @@ namespace IsIdentifiableReviewer.Out { public class SymbolsRulesFactory : IRulePatternFactory { + /// + /// Returns just the failing parts expressed as digits and wrapped in capture group(s) e.g. ^(\d\d-\d\d-\d\d).*([A-Z][A-Z]) + /// + /// + /// + /// public string GetPattern(object sender, Failure failure) { StringBuilder sb = new StringBuilder(); + if (failure.HasOverlappingParts(false)) + return FullStringSymbols(sender, failure); + + foreach (var p in failure.Parts.Distinct().OrderBy(p=>p.Offset)) + { + if (p.Offset == 0) + sb.Append("^"); + + //match with capture group the given Word + sb.Append( "("); + + foreach (char cur in p.Word) + { + if (char.IsDigit(cur)) + sb.Append("\\d"); + else + if (char.IsLetter(cur)) + sb.Append(char.IsUpper(cur) ? "[A-Z]" : "[a-z]"); + else + sb.Append(Regex.Escape(cur.ToString())); + } + + sb.Append(")"); + + if (p.Offset + p.Word.Length == failure.ProblemValue.Length) + sb.Append("$"); + else + sb.Append(".*"); + } + + if(sb.Length == 0) + throw new ArgumentException("Failure had no Parts"); + + + //trim last .* + if (sb.ToString().EndsWith(".*")) + return sb.ToString(0, sb.Length - 2); + + return sb.ToString(); + } + + /// + /// Returns a full symbols match of the entire input string (ProblemValue) + /// + /// + /// + /// + private string FullStringSymbols(object sender, Failure failure) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < failure.ProblemValue.Length; i++) { char cur = failure.ProblemValue[i]; diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs index af3e287c2..82e0188fa 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs @@ -1,4 +1,5 @@ -using IsIdentifiableReviewer.Out; +using System; +using IsIdentifiableReviewer.Out; using Microservices.IsIdentifiable.Failures; using Microservices.IsIdentifiable.Reporting; using NUnit.Framework; @@ -7,20 +8,64 @@ namespace Microservices.IsIdentifiable.Tests.ReviewerTests { public class SymbolsRulesFactoryTests { - - [TestCase("12/34/56",@"^\d\d/\d\d/\d\d$")] - [TestCase("1 6",@"^\d\ \d$")] + [TestCase("MR Head 12-11-20","12-11-20",@"(\d\d-\d\d-\d\d)$")] + [TestCase("CT Head - 12/34/56","12/34/56",@"(\d\d/\d\d/\d\d)$")] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-[A-Z][A-Z][A-Z]-[a-z]\d)")] + [TestCase("123","123",@"^(\d\d\d)$")] + public void TestSymbols_OnePart(string input,string part, string expectedOutput) + { + var f = new SymbolsRulesFactory(); + + var failure = new Failure(new[] {new FailurePart(part, FailureClassification.Person, input.IndexOf(part))}) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + + [TestCase("12 Morton Street","12","eet",@"^(\d\d).*([a-z][a-z][a-z])$")] + [TestCase("Morton MR Smith","MR","Smith",@"([A-Z][A-Z]).*([A-Z][a-z][a-z][a-z][a-z])$")] + public void TestSymbols_TwoParts_NoOverlap(string input,string part1,string part2, string expectedOutput) + { + var f = new SymbolsRulesFactory(); - [TestCase("abc\n123",@"^[a-z][a-z][a-z]\n\d\d\d$")] - public void TestSymbols(string input, string expectedOutput) + var failure = new Failure(new[] + { + new FailurePart(part1, FailureClassification.Person, input.IndexOf(part1)), + new FailurePart(part2, FailureClassification.Person, input.IndexOf(part2)) + }) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + + [TestCase("Clowns","Cl","lowns",@"^[A-Z][a-z][a-z][a-z][a-z][a-z]$")] + public void TestSymbols_TwoParts_Overlap(string input,string part1,string part2, string expectedOutput) + { + var f = new SymbolsRulesFactory(); + + var failure = new Failure(new[] + { + new FailurePart(part1, FailureClassification.Person, input.IndexOf(part1)), + new FailurePart(part2, FailureClassification.Person, input.IndexOf(part2)) + }) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + [Test] + public void TestNoParts() { var f = new SymbolsRulesFactory(); + var ex = Assert.Throws(()=> f.GetPattern(this, new Failure(new FailurePart[0]) {ProblemValue = "fdslkfl;asdf"})); + Assert.AreEqual("Failure had no Parts",ex.Message); - Assert.AreEqual( - expectedOutput, - f.GetPattern(this, new Failure(new FailurePart[0]) {ProblemValue = input}) - ); } } } \ No newline at end of file From e63dee86d87f64fa41afb93f60e64f757d52905d Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 29 May 2020 09:59:31 +0100 Subject: [PATCH 002/138] IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) --- CHANGELOG.md | 1 + .../IsIdentifiableReviewer/MainWindow.cs | 10 ++++--- .../Out/SymbolsRulesFactory.cs | 26 +++++++++++++++++-- .../ReviewerTests/SymbolsRulesFactoryTests.cs | 15 ++++++----- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 070c268a1..3405cabe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Catalogues listed must include one or more column(s) StudyInstanceUID, SeriesInstanceUID, SOPInstanceUID. - Records in the referenced table will blacklist where any UID is found (StudyInstanceUID, SeriesInstanceUID or SOPInstanceUID). This allows blacklisting an entire study or only specific images. - Change the extraction directory generation to be `/image-requests/`. Fixes [MVP Service #159](https://dev.azure.com/smiops/MVP%20Service/_workitems/edit/159/) +- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) ### Fixed diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 33ff4a260..ca228808f 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -542,11 +542,13 @@ public string GetPattern(object sender,Failure failure) var recommendedPattern = defaultFactory.GetPattern(sender,failure); Dictionary buttons = new Dictionary(); - buttons.Add("Clear",""); - buttons.Add("Full",_origIgnorerRulesFactory.GetPattern(sender,failure)); - buttons.Add("Captures",_origUpdaterRulesFactory.GetPattern(sender,failure)); + buttons.Add("x",""); + buttons.Add("F",_origIgnorerRulesFactory.GetPattern(sender,failure)); + buttons.Add("G",_origUpdaterRulesFactory.GetPattern(sender,failure)); - buttons.Add("Symbols",new SymbolsRulesFactory().GetPattern(sender,failure)); + buttons.Add(@"\d",new SymbolsRulesFactory {Mode= SymbolsRuleFactoryMode.DigitsOnly}.GetPattern(sender,failure)); + buttons.Add(@"\c",new SymbolsRulesFactory{Mode= SymbolsRuleFactoryMode.CharactersOnly}.GetPattern(sender,failure)); + buttons.Add(@"\d\c",new SymbolsRulesFactory().GetPattern(sender,failure)); if (GetText("Pattern", "Enter pattern to match failure", recommendedPattern, out string chosen,buttons)) { diff --git a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs index 132e81b78..6bb5805b9 100644 --- a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs @@ -6,8 +6,30 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Determines which bits of a failure get converted to corresponding symbols + /// + public enum SymbolsRuleFactoryMode + { + /// + /// Generates rules that match characters [A-Z]/[a-z] (depending on capitalization of input string) and digits \d + /// + Full, + /// + /// Generates rules that match any digits using \d + /// + DigitsOnly, + + /// + /// Generates rules that match any characters with [A-Z]/[a-z] (depending on capitalization of input string) + /// + CharactersOnly + } + public class SymbolsRulesFactory : IRulePatternFactory { + public SymbolsRuleFactoryMode Mode { get; set; } + /// /// Returns just the failing parts expressed as digits and wrapped in capture group(s) e.g. ^(\d\d-\d\d-\d\d).*([A-Z][A-Z]) /// @@ -31,10 +53,10 @@ public string GetPattern(object sender, Failure failure) foreach (char cur in p.Word) { - if (char.IsDigit(cur)) + if (char.IsDigit(cur) && Mode != SymbolsRuleFactoryMode.CharactersOnly) sb.Append("\\d"); else - if (char.IsLetter(cur)) + if (char.IsLetter(cur) && Mode != SymbolsRuleFactoryMode.DigitsOnly) sb.Append(char.IsUpper(cur) ? "[A-Z]" : "[a-z]"); else sb.Append(Regex.Escape(cur.ToString())); diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs index 82e0188fa..f46724335 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs @@ -9,13 +9,15 @@ namespace Microservices.IsIdentifiable.Tests.ReviewerTests public class SymbolsRulesFactoryTests { - [TestCase("MR Head 12-11-20","12-11-20",@"(\d\d-\d\d-\d\d)$")] - [TestCase("CT Head - 12/34/56","12/34/56",@"(\d\d/\d\d/\d\d)$")] - [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-[A-Z][A-Z][A-Z]-[a-z]\d)")] - [TestCase("123","123",@"^(\d\d\d)$")] - public void TestSymbols_OnePart(string input,string part, string expectedOutput) + [TestCase("MR Head 12-11-20","12-11-20",@"(\d\d-\d\d-\d\d)$",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 12/34/56","12/34/56",@"(\d\d/\d\d/\d\d)$",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-[A-Z][A-Z][A-Z]-[a-z]\d)",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(123-[A-Z][A-Z][A-Z]-[a-z]4)",SymbolsRuleFactoryMode.CharactersOnly)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-ABC-n\d)",SymbolsRuleFactoryMode.DigitsOnly)] + [TestCase("123","123",@"^(\d\d\d)$",SymbolsRuleFactoryMode.Full)] + public void TestSymbols_OnePart(string input,string part, string expectedOutput,SymbolsRuleFactoryMode mode) { - var f = new SymbolsRulesFactory(); + var f = new SymbolsRulesFactory(){Mode = mode}; var failure = new Failure(new[] {new FailurePart(part, FailureClassification.Person, input.IndexOf(part))}) { @@ -25,6 +27,7 @@ public void TestSymbols_OnePart(string input,string part, string expectedOutput) Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); } + [TestCase("12 Morton Street","12","eet",@"^(\d\d).*([a-z][a-z][a-z])$")] [TestCase("Morton MR Smith","MR","Smith",@"([A-Z][A-Z]).*([A-Z][a-z][a-z][a-z][a-z])$")] public void TestSymbols_TwoParts_NoOverlap(string input,string part1,string part2, string expectedOutput) From 14f3dafa3f9aaf162afe0336f371cd8da8fcef5b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2020 10:39:45 +0100 Subject: [PATCH 003/138] Bump Microsoft.CodeAnalysis.CSharp.Scripting from 3.6.0 to 3.7.0 Bump Microsoft.CodeAnalysis.CSharp.Scripting from 3.6.0 to 3.7.0 --- PACKAGES.md | 2 +- appveyor.yml | 3 +++ .../Microservices.CohortExtractor.csproj | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 16004cb8f..02a498fb9 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -18,7 +18,7 @@ | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.6](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | -| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.6.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.6.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | +| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | | Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.6](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6/License) | Caching ID mappings retrieved from Redis/MySQL | | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0/License) | For writting/reading dicom tags into MongoDb databases| | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | diff --git a/appveyor.yml b/appveyor.yml index 68b333e6c..21b9ff67a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,6 +15,9 @@ before_build: - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" - cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.initiate()'" +- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.conf()'" +- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.status()'" +- cmd: "appveyor exit" - choco install opencover.portable - choco install rabbitmq diff --git a/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj b/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj index a91e9e7f0..0d8799eb8 100644 --- a/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj +++ b/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj @@ -26,7 +26,7 @@ - + From 4f929be86eaac401fad9b1aafa6576559d0778c5 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Thu, 6 Aug 2020 10:43:18 +0100 Subject: [PATCH 004/138] Make Mongo show status output instead of dumping it --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 21b9ff67a..28147d203 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,9 +14,9 @@ before_build: - cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.initiate()'" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.conf()'" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.status()'" +- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.initiate())'" +- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.conf())'" +- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.status())'" - cmd: "appveyor exit" - choco install opencover.portable - choco install rabbitmq From be78847142cb5219f77c7c50d55d3e13eeb0d9b5 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Thu, 6 Aug 2020 11:05:56 +0100 Subject: [PATCH 005/138] Change Mongo quoting --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 28147d203..322cfe8db 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,9 +14,9 @@ before_build: - cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.initiate())'" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.conf())'" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'printjson(rs.status())'" +- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.initiate())\"" +- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.conf())\"" +- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.status())\"" - cmd: "appveyor exit" - choco install opencover.portable - choco install rabbitmq From 5916fceed901647a98d2b20eb9ad6d5cf5aa1087 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Thu, 6 Aug 2020 11:08:02 +0100 Subject: [PATCH 006/138] Re-enable appveyor now Mongo is fixed --- appveyor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 322cfe8db..888ba7179 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,9 +15,6 @@ before_build: - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" - cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.initiate())\"" -- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.conf())\"" -- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.status())\"" -- cmd: "appveyor exit" - choco install opencover.portable - choco install rabbitmq From aad39f277d235990cffa03e949e4682e9a3f0fd1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2020 14:09:50 +0100 Subject: [PATCH 007/138] Bump Microsoft.NET.Test.Sdk from 16.6.1 to 16.7.0 Bump Microsoft.NET.Test.Sdk from 16.6.1 to 16.7.0 --- .../Applications.DicomDirectoryProcessor.Tests.csproj | 2 +- .../Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj | 2 +- tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj | 2 +- .../Microservices.CohortExtractor.Tests.csproj | 2 +- .../Microservices.CohortPackager.Tests.csproj | 2 +- .../Microservices.DeadLetterReprocessor.Tests.csproj | 2 +- .../Microservices.DicomRelationalMapper.Tests.csproj | 2 +- .../Microservices.DicomReprocessor.Tests.csproj | 2 +- .../Microservices.DicomTagReader.Tests.csproj | 2 +- .../Microservices.IdentifierMapper.Tests.csproj | 2 +- .../Microservices.IsIdentifiable.Tests.csproj | 2 +- .../Microservices.MongoDBPopulator.Tests.csproj | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj index 0dc51d917..8826c601d 100644 --- a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj index 098c117c3..a004ebbd1 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj +++ b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index 09b595e08..b38f8c804 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -33,7 +33,7 @@ - + diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj index cb8e2240c..44a18f3c0 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj index cfefe690f..32be23a42 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj +++ b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj @@ -13,7 +13,7 @@ true - + all diff --git a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj index 1460802e2..91a37b880 100644 --- a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj index d50305355..bf79aba88 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj index 8725ca361..5313be9cd 100644 --- a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj index 5fc19dfa1..9f363b27a 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj +++ b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index cb30ae879..23c9354bd 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj index 69b724b3b..7cf7a1af7 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj index 53807b78a..37415c9e5 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 2b964db95d8fa46de09dc630a4f3b5a84c849ff4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2020 13:33:37 +0100 Subject: [PATCH 008/138] Bump fo-dicom.NetCore from 4.0.5 to 4.0.6 Bump fo-dicom.NetCore from 4.0.5 to 4.0.6 --- PACKAGES.md | 2 +- src/common/Smi.Common/Smi.Common.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 02a498fb9..e8a293485 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -11,7 +11,7 @@ | ------- | ------------| --------| ------- | ------- | -------------------------- | | CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.8.0](https://www.nuget.org/packages/CommandLineParser/2.8.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | | CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.5](https://www.nuget.org/packages/CsvHelper/15.0.5) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | -| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.5](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.5) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | +| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.2](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.6](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 0333b6580..57bad6248 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -31,7 +31,7 @@ - + From 67b7157b5542e0767a3195a15f2db21f69296e8f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2020 16:58:24 +0100 Subject: [PATCH 009/138] Bump fo-dicom.Drawing from 4.0.5 to 4.0.6 Bump fo-dicom.Drawing from 4.0.5 to 4.0.6 --- PACKAGES.md | 2 +- .../Microservices.IsIdentifiable.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index e8a293485..527aa0730 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -33,6 +33,6 @@ | Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.81.0](https://www.nuget.org/packages/Terminal.Gui/0.81.0) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | | | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | | YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [8.1.2](https://www.nuget.org/packages/YamlDotNet/8.1.2) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.5](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.5) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | +| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | | coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [1.4.2](https://www.nuget.org/packages/coveralls.io/1.4.2) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io | | | OpenCover | [GitHub](https://github.com/OpenCover/opencover) | [4.7.922](https://www.nuget.org/packages/OpenCover/4.7.922) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE) | Calculates code coverage for tests| | diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index 498d5597a..fd7ca464a 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -41,7 +41,7 @@ - + From 1741075cc0ee6043d930ec3f65cf9c9195554fc8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:51:28 +0100 Subject: [PATCH 010/138] Bump Microsoft.Extensions.Caching.Memory from 3.1.6 to 3.1.7 Bump Microsoft.Extensions.Caching.Memory from 3.1.6 to 3.1.7 --- PACKAGES.md | 2 +- .../Microservices.IdentifierMapper.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 527aa0730..6bee87d98 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -19,7 +19,7 @@ | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | -| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.6](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6/License) | Caching ID mappings retrieved from Redis/MySQL | +| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.7](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL | | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0/License) | For writting/reading dicom tags into MongoDb databases| | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | diff --git a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj index ef0405e27..5b4597df3 100644 --- a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj +++ b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj @@ -19,7 +19,7 @@ - + From a3687a9910ca8519739358e7bfb6d5e646d0ee25 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2020 09:05:46 +0100 Subject: [PATCH 011/138] Bump maven-resources-plugin from 3.1.0 to 3.2.0 Bump maven-resources-plugin from 3.1.0 to 3.2.0 --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 6c4302e67..bb7113ea3 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -86,7 +86,7 @@ maven-resources-plugin - 3.1.0 + 3.2.0 From 23198008b93b474f4fb8307c725fe7117fd89228 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Wed, 12 Aug 2020 13:38:48 +0100 Subject: [PATCH 012/138] Disable publishtrimmed since it breaks things --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc5ffe5c8..0c8c75843 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ jobs: mkdir -p ./dist for platform in linux win do - dotnet publish -p:PublishTrimmed=true -c Release -r $platform-x64 -o $(pwd)/dist/$platform-x64 -nologo -v q + dotnet publish -p:PublishTrimmed=false -c Release -r $platform-x64 -o $(pwd)/dist/$platform-x64 -nologo -v q done ( cd dist && zip -9r smi-services-win-x64-$TRAVIS_TAG.zip ./win-x64 && tar cf - ./linux-x64 | pigz -3 > smi-services-linux-x64-$TRAVIS_TAG.tar.gz && cd - ) mvn -ntp -q -f src/common/com.smi.microservices.parent/pom.xml install assembly:single@create-deployable -DskipTests From 2f60feaaae2b3b997de44706717d9c4be590a9a2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 07:06:45 +0000 Subject: [PATCH 013/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.4.6 to 3.5.0. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.4.6...v3.5.0) Signed-off-by: dependabot-preview[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index bb7113ea3..f168ba63b 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.4.6 + 3.5.0 From 3cb0cb758c27556a9cac2412be7aa4d2c3cbb061 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 10:02:46 +0000 Subject: [PATCH 014/138] Bump HIC.RDMP.Dicom from 2.1.6 to 2.1.7 (#352) --- PACKAGES.md | 6 +++--- src/common/Smi.Common/Smi.Common.csproj | 4 ++-- .../Microservices.DicomRelationalMapper.csproj | 2 +- tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj | 2 +- .../Microservices.IdentifierMapper.Tests.csproj | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 6bee87d98..60038e45d 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -13,9 +13,9 @@ | CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.5](https://www.nuget.org/packages/CsvHelper/15.0.5) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | | fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | -| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.2](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.6](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | -| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.6](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | +| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | +| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.7](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 57bad6248..c5ed12959 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -34,8 +34,8 @@ - - + + diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index a0b86b30c..4c8191e0d 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -18,7 +18,7 @@ - + diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index b38f8c804..3ac6e498c 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -34,7 +34,7 @@ - + diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index 23c9354bd..ee875d4c1 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -13,7 +13,7 @@ true - + all From 88268015ea26fd7b53ac9d4c5eee2f6f3ad86e1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 12:17:43 +0100 Subject: [PATCH 015/138] Bump HIC.RDMP.Plugin from 4.1.7 to 4.1.8 Bump HIC.RDMP.Plugin from 4.1.7 to 4.1.8 --- PACKAGES.md | 2 +- src/common/Smi.Common/Smi.Common.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 60038e45d..05429479e 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -15,7 +15,7 @@ | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | -| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.7](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | +| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index c5ed12959..ccaa6159f 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -35,7 +35,7 @@ - + From b9565800f121476bb8a08ef33b23db3e4897d940 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 13:18:00 +0100 Subject: [PATCH 016/138] Bump HIC.RDMP.Plugin.Test from 4.1.7 to 4.1.8 Bump HIC.RDMP.Plugin.Test from 4.1.7 to 4.1.8 --- tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj | 2 +- .../Microservices.IdentifierMapper.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index 3ac6e498c..1de67b065 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -34,7 +34,7 @@ - + diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index ee875d4c1..5ff482ee0 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -13,7 +13,7 @@ true - + all From 7e66293f9337932b79d22eec3a57dbd05b42ff5c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 13:49:47 +0100 Subject: [PATCH 017/138] Bump HIC.DicomTypeTranslation from 2.3.0 to 2.3.1 Bump HIC.DicomTypeTranslation from 2.3.0 to 2.3.1 --- PACKAGES.md | 2 +- src/common/Smi.Common/Smi.Common.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 05429479e..0418822fb 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -12,7 +12,7 @@ | CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.8.0](https://www.nuget.org/packages/CommandLineParser/2.8.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | | CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.5](https://www.nuget.org/packages/CsvHelper/15.0.5) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | | fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | -| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | +| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index ccaa6159f..b355b95e4 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -33,7 +33,7 @@ - + From f918b2b121adcd712fc066ee4ad8f464a9227d79 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 18 Aug 2020 10:51:18 +0000 Subject: [PATCH 018/138] exclude fo-dicom from dependabot updates --- .dependabot/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dependabot/config.yml b/.dependabot/config.yml index 13e01f37a..b8fbbce99 100644 --- a/.dependabot/config.yml +++ b/.dependabot/config.yml @@ -7,6 +7,9 @@ update_configs: default_reviewers: - rkm - tznind + ignored_updates: + - match: + dependency_name: "fo-dicom*" - package_manager: "java:maven" directory: "/src/common/com.smi.microservices.parent/" update_schedule: "daily" From a2c944d5809a4b6ef29e71c72bb2cfad6b95dc21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2020 12:00:09 +0100 Subject: [PATCH 019/138] Update Dependabot config file (#356) * Update Dependabot config file * re-add ignore for fo-dicom * fix schema difference Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: Ruairidh MacLeod --- .dependabot/config.yml | 19 ------------------- .github/dependabot.yml | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) delete mode 100644 .dependabot/config.yml create mode 100644 .github/dependabot.yml diff --git a/.dependabot/config.yml b/.dependabot/config.yml deleted file mode 100644 index b8fbbce99..000000000 --- a/.dependabot/config.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 1 -update_configs: - - package_manager: "dotnet:nuget" - directory: "/" - update_schedule: "live" - target_branch: "develop" - default_reviewers: - - rkm - - tznind - ignored_updates: - - match: - dependency_name: "fo-dicom*" - - package_manager: "java:maven" - directory: "/src/common/com.smi.microservices.parent/" - update_schedule: "daily" - target_branch: "develop" - default_reviewers: - - rkm - - tznind diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..0e5e918a1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +version: 2 +updates: +- package-ecosystem: nuget + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 99 + target-branch: develop + reviewers: + - rkm + - tznind + ignore: + - dependency-name: "fo-dicom*" +- package-ecosystem: maven + directory: "/src/common/com.smi.microservices.parent" + schedule: + interval: daily + open-pull-requests-limit: 99 + target-branch: develop + reviewers: + - rkm + - tznind From 7099bdb99cb2562a3e81d915638d28d690931819 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2020 11:03:44 +0000 Subject: [PATCH 020/138] Bump amqp-client in /src/common/com.smi.microservices.parent Bumps [amqp-client](https://github.com/rabbitmq/rabbitmq-java-client) from 5.2.0 to 5.9.0. - [Release notes](https://github.com/rabbitmq/rabbitmq-java-client/releases) - [Commits](https://github.com/rabbitmq/rabbitmq-java-client/compare/v5.2.0...v5.9.0) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index f168ba63b..4e50ac175 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -154,7 +154,7 @@ com.rabbitmq amqp-client - 5.2.0 + 5.9.0 From 8d97c0e356ab04c481fe980f424692f3f5f258ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2020 15:19:05 +0100 Subject: [PATCH 021/138] Bump jackson-databind in /src/common/com.smi.microservices.parent (#357) Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.10.1 to 2.11.2. --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index f168ba63b..57365d4e1 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -224,7 +224,7 @@ com.fasterxml.jackson.core jackson-databind - 2.10.1 + 2.11.2 From 2ae6e5a425572486c9d779b50c9353d6443ad0b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2020 21:28:17 +0000 Subject: [PATCH 022/138] Bump HIC.BadMedicine.Dicom from 0.0.6 to 0.0.7 Bumps [HIC.BadMedicine.Dicom](https://github.com/HicServices/BadMedicine.Dicom) from 0.0.6 to 0.0.7. - [Release notes](https://github.com/HicServices/BadMedicine.Dicom/releases) - [Changelog](https://github.com/HicServices/BadMedicine.Dicom/blob/master/CHANGELOG.md) - [Commits](https://github.com/HicServices/BadMedicine.Dicom/compare/v0.0.6...0.0.7) Signed-off-by: dependabot[bot] --- tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index 1de67b065..9ecb8d403 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -32,7 +32,7 @@ - + From ab4d864f4d09e47de394bbc42f3f71da6d61fff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2020 05:00:48 +0000 Subject: [PATCH 023/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.0 to 3.5.2. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.0...v3.5.2) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 49abd3659..9568fc5c6 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.0 + 3.5.2 From 0ab49445d1258f6eed2677a637cd468535bbe186 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 06:11:40 +0000 Subject: [PATCH 024/138] Bump Microsoft.NET.Test.Sdk from 16.7.0 to 16.7.1 Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 16.7.0 to 16.7.1. - [Release notes](https://github.com/microsoft/vstest/releases) - [Commits](https://github.com/microsoft/vstest/compare/v16.7.0...v16.7.1) Signed-off-by: dependabot[bot] --- .../Applications.DicomDirectoryProcessor.Tests.csproj | 2 +- .../Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj | 2 +- tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj | 2 +- .../Microservices.CohortExtractor.Tests.csproj | 2 +- .../Microservices.CohortPackager.Tests.csproj | 2 +- .../Microservices.DeadLetterReprocessor.Tests.csproj | 2 +- .../Microservices.DicomRelationalMapper.Tests.csproj | 2 +- .../Microservices.DicomReprocessor.Tests.csproj | 2 +- .../Microservices.DicomTagReader.Tests.csproj | 2 +- .../Microservices.IdentifierMapper.Tests.csproj | 2 +- .../Microservices.IsIdentifiable.Tests.csproj | 2 +- .../Microservices.MongoDBPopulator.Tests.csproj | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj index 8826c601d..99408ef2f 100644 --- a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj index a004ebbd1..0cb9206a6 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj +++ b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index 9ecb8d403..e194182a0 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -33,7 +33,7 @@ - + diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj index 44a18f3c0..e5407fb94 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj index 32be23a42..23a4e0a5b 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj +++ b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj @@ -13,7 +13,7 @@ true - + all diff --git a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj index 91a37b880..86290491d 100644 --- a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj index bf79aba88..b5400cc61 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj index 5313be9cd..399841548 100644 --- a/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DicomReprocessor.Tests/Microservices.DicomReprocessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj index 9f363b27a..22ab4b977 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj +++ b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index 5ff482ee0..3d1fad127 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj index 7cf7a1af7..da4b5766f 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj index 37415c9e5..fbc233326 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive From bee6cc5fe1d9c21f2d051ca1a423923c2343fd09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 05:01:54 +0000 Subject: [PATCH 025/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.2 to 3.5.5. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.2...v3.5.5) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 9568fc5c6..560c97e25 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.2 + 3.5.5 From cf39a285874987193a74bdf12393729a47dbb3de Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 24 Aug 2020 17:25:26 +0000 Subject: [PATCH 026/138] Feature/security code scan (#366) * Add SecurityCodeScan to build process * Note addition of SecurityCodeScan package in CHANGELOG.md --- CHANGELOG.md | 2 ++ PACKAGES.md | 1 + src/common/Smi.Common/Smi.Common.csproj | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f698018d..2b83c7db6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +- Add SecurityCodeScan tool to build chain for .Net code + ## [1.11.1] - 2020-08-12 - Set PublishTrimmed to false to fix bug with missing assemblies in prod. diff --git a/PACKAGES.md b/PACKAGES.md index 0418822fb..174039691 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -24,6 +24,7 @@ | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | | RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | +| SecurityCodeScan | [GitHub](https://security-code-scan.github.io/) | [3.5.3](https://www.nuget.org/packages/SecurityCodeScan/3.5.3) | [LGPL 3.0](https://opensource.org/licenses/lgpl-3.0.html) | Scans code for security issues during build | | | StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.1.58](https://www.nuget.org/packages/StackExchange.Redis/2.1.58) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | | | Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | | | System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Drawing.Common/4.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | | diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index b355b95e4..ba04fa0cd 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -40,6 +40,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + From 97ad6a088db964705dfbc83fc93540ac875aaf16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2020 05:12:26 +0000 Subject: [PATCH 027/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.5 to 3.5.6. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.5...v3.5.6) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 560c97e25..94d520473 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.5 + 3.5.6 From 31d2d313b696b5dfb452b52f5c35d57f9e665593 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2020 06:12:48 +0000 Subject: [PATCH 028/138] Bump Terminal.Gui from 0.81.0 to 0.89.4 Bumps [Terminal.Gui](https://github.com/migueldeicaza/gui.cs) from 0.81.0 to 0.89.4. - [Release notes](https://github.com/migueldeicaza/gui.cs/releases) - [Commits](https://github.com/migueldeicaza/gui.cs/commits/v0.89.4) Signed-off-by: dependabot[bot] --- .../IsIdentifiableReviewer/IsIdentifiableReviewer.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj index c9d009e11..08babf085 100644 --- a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj +++ b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj @@ -33,7 +33,7 @@ - + From 62cd838689a1345121835960984443a37094a22e Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 25 Aug 2020 09:03:14 +0100 Subject: [PATCH 029/138] Update PACKAGES.md --- PACKAGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PACKAGES.md b/PACKAGES.md index 174039691..b3501bdc2 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -31,7 +31,7 @@ | System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [12.1.1](https://www.nuget.org/packages/System.IO.Abstractions/12.1.1) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | | | System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | File I/O | | | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | -| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.81.0](https://www.nuget.org/packages/Terminal.Gui/0.81.0) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | | +| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.89.4](https://www.nuget.org/packages/Terminal.Gui/0.89.4) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | | | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | | YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [8.1.2](https://www.nuget.org/packages/YamlDotNet/8.1.2) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| | fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | From 682609e6abf0be598a0e1c99f2d46bdda236404d Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 25 Aug 2020 10:20:11 +0100 Subject: [PATCH 030/138] Updated to match new API --- .../IsIdentifiableReviewer/MainWindow.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 33ff4a260..9e548b4b7 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -36,6 +36,14 @@ class MainWindow : View,IRulePatternFactory Stack History = new Stack(); + ColorScheme _greyOnBlack = new ColorScheme() + { + Normal = Attribute.Make(Color.Black,Color.Gray), + HotFocus = Attribute.Make(Color.Black,Color.Gray), + Disabled = Attribute.Make(Color.Black,Color.Gray), + Focus = Attribute.Make(Color.Black,Color.Gray), + }; + public MainWindow(List targets, IsIdentifiableReviewerOptions opts, IgnoreRuleGenerator ignorer, RowUpdater updater) { _targets = targets; @@ -66,7 +74,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno Height = 1 }; - _info.TextColor = Attribute.Make(Color.Black,Color.Gray); + _info.ColorScheme = _greyOnBlack; _valuePane = new ValuePane() { @@ -101,7 +109,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno X=28, Width = 5 }; - _gotoTextField.Changed += (s,e) => GoTo(); + _gotoTextField.TextChanged = (s) => GoTo(); frame.Add(_gotoTextField); frame.Add(new Label(23,0,"GoTo:")); @@ -133,7 +141,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno frame.Add(_updateRuleLabel); var cbCustomPattern = new CheckBox(23,1,"Custom Patterns",false); - cbCustomPattern.Toggled += (c, s) => + cbCustomPattern.Toggled = (b) => { Updater.RulesFactory = cbCustomPattern.Checked ? this : _origUpdaterRulesFactory; Ignorer.RulesFactory = cbCustomPattern.Checked ? this : _origIgnorerRulesFactory; @@ -143,7 +151,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno _cbRulesOnly = new CheckBox(23,2,"Rules Only",opts.OnlyRules); Updater.RulesOnly = opts.OnlyRules; - _cbRulesOnly.Toggled += (c, s) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; + _cbRulesOnly.Toggled += (b) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; frame.Add(_cbRulesOnly); top.Add (menu); From a828fd34eb6dd1bb92b3fec3ad623ee077253f6a Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 25 Aug 2020 10:22:24 +0100 Subject: [PATCH 031/138] Set Toggled to = in both cases --- src/applications/IsIdentifiableReviewer/MainWindow.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 9e548b4b7..15b15200a 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -151,7 +151,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno _cbRulesOnly = new CheckBox(23,2,"Rules Only",opts.OnlyRules); Updater.RulesOnly = opts.OnlyRules; - _cbRulesOnly.Toggled += (b) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; + _cbRulesOnly.Toggled = (b) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; frame.Add(_cbRulesOnly); top.Add (menu); From 75eb5a4d4730133a584c473831a8a75820793f92 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 16:07:01 +0100 Subject: [PATCH 032/138] Add debug message --- .../Execution/JobProcessing/ExtractJobWatcher.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs index c7ba49f58..797312d57 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs @@ -126,6 +126,8 @@ private void DoJobCompletionTasks(ExtractJobInfo jobInfo) _jobStore.MarkJobCompleted(jobId); _reporter.CreateReport(jobId); + _logger.Info($"Report for {jobId} created"); + _notifier.NotifyJobCompleted(jobInfo); } } From 371a6317e31013662efb55a0dde31cc2aa50931a Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 16:15:02 +0100 Subject: [PATCH 033/138] Improve extraction report sections --- .../JobProcessing/Reporting/JobReporterBase.cs | 6 ++++-- .../JobProcessing/Reporting/JobReporterTest.cs | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs index 11944b15b..b24a0d781 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs @@ -81,6 +81,8 @@ private static IEnumerable JobHeader(ExtractJobInfo jobInfo) "", "Report contents:", "- Verification failures", + " - Summary", + " - Full Details", "- Rejected failures", "- Anonymisation failures", }; @@ -132,7 +134,7 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume } } - streamWriter.WriteLine("Summary:"); + streamWriter.WriteLine("### Summary"); streamWriter.WriteLine(); var sb = new StringBuilder(); @@ -160,7 +162,7 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume // Now write-out the same, but with the file listing streamWriter.WriteLine(); - streamWriter.WriteLine("Full details:"); + streamWriter.WriteLine("### Full details"); streamWriter.WriteLine(); streamWriter.Write(sb); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs index 7871d4d78..7dffdbd32 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs @@ -110,15 +110,17 @@ public void Test_JobReporterBase_CreateReport_Empty() Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary -Full details: +### Full details ## Rejected files @@ -204,18 +206,20 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary - Tag: ScanOptions (1 total occurrence(s)) - Value: 'FOO' (1 occurrence(s)) -Full details: +### Full details - Tag: ScanOptions (1 total occurrence(s)) - Value: 'FOO' (1 occurrence(s)) @@ -373,12 +377,14 @@ public void Test_JobReporterBase_CreateReport_AggregateData() Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary - Tag: ScanOptions (3 total occurrence(s)) - Value: 'FOO' (2 occurrence(s)) @@ -388,7 +394,7 @@ public void Test_JobReporterBase_CreateReport_AggregateData() - Value: 'BAZ' (2 occurrence(s)) -Full details: +### Full details - Tag: ScanOptions (3 total occurrence(s)) - Value: 'FOO' (2 occurrence(s)) From 9be29af4b43522c86be8c4ba1592c75e126c62e2 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 17:05:18 +0100 Subject: [PATCH 034/138] Extraction report: Group PixelData separately and sort by length --- CHANGELOG.md | 3 +- .../Reporting/JobReporterBase.cs | 54 ++++++-- .../Reporting/JobReporterTest.cs | 131 +++++++++++++++++- 3 files changed, 168 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b83c7db6..46a715d09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -- Add SecurityCodeScan tool to build chain for .Net code +- Add SecurityCodeScan tool to build chain for .Net code +- Extraction report: Group PixelData separately and sort by length ## [1.11.1] - 2020-08-12 diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs index b24a0d781..ad1d7b758 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs @@ -140,24 +140,24 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume var sb = new StringBuilder(); // Write-out the groupings, ordered by descending count, as a summary without the list of associated files - List>>> grouped = groupedFailures.OrderByDescending(x => x.Value.Sum(y => y.Value.Count)).ToList(); + // Ignore the pixel data here since we deal with it separately below + const string pixelData = "PixelData"; + List>>> grouped = groupedFailures + .Where(x => x.Key != pixelData) + .OrderByDescending(x => x.Value.Sum(y => y.Value.Count)) + .ToList(); + foreach ((string tag, Dictionary> failures) in grouped) { - int totalOccurrences = failures.Sum(x => x.Value.Count); - string line = $"- Tag: {tag} ({totalOccurrences} total occurrence(s))"; - streamWriter.WriteLine(line); - sb.AppendLine(line); - foreach ((string problemVal, List relatedFiles) in failures.OrderByDescending(x => x.Value.Count)) - { - line = $" - Value: '{problemVal}' ({relatedFiles.Count} occurrence(s))"; - streamWriter.WriteLine(line); - sb.AppendLine(line); - foreach (string file in relatedFiles) - sb.AppendLine($" - {file}"); - } + WriteVerificationValuesTag(tag, failures, streamWriter, sb); + WriteVerificationValues(failures.OrderByDescending(x => x.Value.Count), streamWriter, sb); + } - streamWriter.WriteLine(); - sb.AppendLine(); + // Now list the pixel data, which we instead order by decreasing length + if (groupedFailures.TryGetValue(pixelData, out Dictionary> pixelFailures)) + { + WriteVerificationValuesTag(pixelData, pixelFailures, streamWriter, sb); + WriteVerificationValues(pixelFailures.OrderByDescending(x => x.Key.Length), streamWriter, sb); } // Now write-out the same, but with the file listing @@ -167,6 +167,30 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume streamWriter.Write(sb); } + private static void WriteVerificationValuesTag(string tag, Dictionary> failures, TextWriter streamWriter, StringBuilder sb) + { + int totalOccurrences = failures.Sum(x => x.Value.Count); + string line = $"- Tag: {tag} ({totalOccurrences} total occurrence(s))"; + streamWriter.WriteLine(line); + sb.AppendLine(line); + } + + private static void WriteVerificationValues(IEnumerable>> values, TextWriter streamWriter, StringBuilder sb) + { + + foreach ((string problemVal, List relatedFiles) in values) + { + string line = $" - Value: '{problemVal}' ({relatedFiles.Count} occurrence(s))"; + streamWriter.WriteLine(line); + sb.AppendLine(line); + foreach (string file in relatedFiles) + sb.AppendLine($" - {file}"); + } + + streamWriter.WriteLine(); + sb.AppendLine(); + } + protected abstract void ReleaseUnmanagedResources(); public abstract void Dispose(); ~JobReporterBase() => ReleaseUnmanagedResources(); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs index 7dffdbd32..690448d98 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs @@ -170,8 +170,8 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() [ { 'Parts': [], - 'Resource': '/foo1.dcm', - 'ResourcePrimaryKey': '1.2.3.4', + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', 'ProblemField': 'ScanOptions', 'ProblemValue': 'FOO' } @@ -356,8 +356,7 @@ public void Test_JobReporterBase_CreateReport_AggregateData() mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); mockJobStore.Setup(x => x.GetCompletedJobRejections(It.IsAny())).Returns(new List>>()); mockJobStore.Setup(x => x.GetCompletedJobAnonymisationFailures(It.IsAny())).Returns(new List>()); - mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())) - .Returns(verificationFailures); + mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())).Returns(verificationFailures); TestJobReporter reporter; using (reporter = new TestJobReporter(mockJobStore.Object)) @@ -420,6 +419,130 @@ public void Test_JobReporterBase_CreateReport_AggregateData() TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); Assert.True(reporter.Disposed); } + + + [Test] + public void Test_JobReporterBase_CreateReport_WithPixelData() + { + // NOTE(rkm 2020-08-25) Tests that the "Z" tag is ordered before PixelData, and that PixelData items are ordered by decreasing length not by occurrence + + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed); + + + const string report = @" +[ + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'aaaaaaaaaaa' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'a' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'a' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'Z', + 'ProblemValue': 'bar' + }, +]"; + + var verificationFailures = new List> + { + new Tuple("foo1.dcm", report), + }; + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobRejections(It.IsAny())).Returns(new List>>()); + mockJobStore.Setup(x => x.GetCompletedJobAnonymisationFailures(It.IsAny())).Returns(new List>()); + mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())).Returns(verificationFailures); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 + +Report contents: +- Verification failures + - Summary + - Full Details +- Rejected failures +- Anonymisation failures + +## Verification failures + +### Summary + +- Tag: Z (1 total occurrence(s)) + - Value: 'bar' (1 occurrence(s)) + +- Tag: PixelData (3 total occurrence(s)) + - Value: 'aaaaaaaaaaa' (1 occurrence(s)) + - Value: 'a' (2 occurrence(s)) + + +### Full details + +- Tag: Z (1 total occurrence(s)) + - Value: 'bar' (1 occurrence(s)) + - foo1.dcm + +- Tag: PixelData (3 total occurrence(s)) + - Value: 'aaaaaaaaaaa' (1 occurrence(s)) + - foo1.dcm + - Value: 'a' (2 occurrence(s)) + - foo1.dcm + - foo1.dcm + + +## Rejected files + + +## Anonymisation failures + + +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } + } #endregion From d5e685f55fa421cdc22845960dfa495f9328738b Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 18:45:49 +0100 Subject: [PATCH 035/138] Reference CommandLineParser centrally --- src/common/Smi.Common/Smi.Common.csproj | 1 + .../Microservices.IsIdentifiable.csproj | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index ba04fa0cd..93de313f5 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -31,6 +31,7 @@ + diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index fd7ca464a..c550d1e1f 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -39,7 +39,6 @@ - From c271af2dc7c92167d6100f94c6054ac9370874c4 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 18:52:24 +0100 Subject: [PATCH 036/138] Add FileCopier base --- SmiServices.sln | 9 ++++++- .../Microservices.FileCopier.csproj | 19 +++++++++++++ .../Microservices.FileCopier/Program.cs | 27 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj create mode 100644 src/microservices/Microservices.FileCopier/Program.cs diff --git a/SmiServices.sln b/SmiServices.sln index a34da7b34..289fccd3a 100644 --- a/SmiServices.sln +++ b/SmiServices.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.29411.108 @@ -75,6 +75,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiabl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IsIdentifiableReviewer", "src\applications\IsIdentifiableReviewer\IsIdentifiableReviewer.csproj", "{C2031E86-81B4-405A-A923-9B82E0CE196F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -181,6 +183,10 @@ Global {C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.Build.0 = Debug|x64 {C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.ActiveCfg = Release|x64 {C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.Build.0 = Release|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.ActiveCfg = Debug|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.Build.0 = Debug|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.ActiveCfg = Release|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -211,6 +217,7 @@ Global {1A27E9E8-F16E-43F9-927E-5FE92E2F97D8} = {421CCD37-3817-4748-B184-A134E19DD75C} {E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2} = {421CCD37-3817-4748-B184-A134E19DD75C} {C2031E86-81B4-405A-A923-9B82E0CE196F} = {8B943F2C-835B-484A-86D2-3F1462970605} + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7} = {421CCD37-3817-4748-B184-A134E19DD75C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {11CDEA53-71E8-4A9B-BC0D-74F4EB54F740} diff --git a/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj b/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj new file mode 100644 index 000000000..3c185c987 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj @@ -0,0 +1,19 @@ + + FileCopier + Exe + netcoreapp3.1 + false + bin\$(Platform)\$(Configuration)\ + x64 + true + 8.0 + full + true + + + + + + + + diff --git a/src/microservices/Microservices.FileCopier/Program.cs b/src/microservices/Microservices.FileCopier/Program.cs new file mode 100644 index 000000000..3adac6abc --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Program.cs @@ -0,0 +1,27 @@ +using CommandLine; +using Smi.Common.Options; + + +namespace Microservices.FileCopier +{ + internal static class Program + { + /// + /// Program entry point when run from the command line + /// + /// + private static int Main(string[] args) + { + return Parser.Default.ParseArguments(args).MapResult( + (o) => + { + GlobalOptions options = GlobalOptions.Load(o); + + //var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomTagReaderHost(options)); + //return bootstrapper.Main(); + return 0; + }, + err => -100); + } + } +} From 350a54274b2bf071c864522dc988f49c7c6fcaa8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 19:48:58 +0100 Subject: [PATCH 037/138] Implement file copier --- branch_todo.md | 5 ++ .../Messages/Extraction/ExtractFileStatus.cs | 14 +++- .../Extraction/ExtractFileStatusMessage.cs | 6 +- .../Smi.Common/Options/GlobalOptions.cs | 9 +++ .../Execution/FileCopier.cs | 78 +++++++++++++++++++ .../Execution/FileCopierHost.cs | 36 +++++++++ .../Execution/IFileCopier.cs | 11 +++ .../Messaging/FileCopyQueueConsumer.cs | 39 ++++++++++ .../Microservices.FileCopier/Program.cs | 10 +-- 9 files changed, 201 insertions(+), 7 deletions(-) create mode 100644 branch_todo.md create mode 100644 src/microservices/Microservices.FileCopier/Execution/FileCopier.cs create mode 100644 src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs create mode 100644 src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs create mode 100644 src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs diff --git a/branch_todo.md b/branch_todo.md new file mode 100644 index 000000000..aed9ba326 --- /dev/null +++ b/branch_todo.md @@ -0,0 +1,5 @@ + +# Branch TODO + +- Update the extraction plan doc +- Ensure changes to message definitions are reflected in the Java code \ No newline at end of file diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs index e016dd727..f0c60b463 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs @@ -19,6 +19,18 @@ public enum ExtractFileStatus /// /// The file could not be anonymised and will not be retired /// - ErrorWontRetry + ErrorWontRetry, + + // TODO Java + /// + /// The source file could not be found under the given filesystem root + /// + FileMissing, + + // TODO Java + /// + /// The source file was successfully copied to the destination + /// + Copied, } } diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs index 54f0b4d64..c2468ed66 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs @@ -1,4 +1,4 @@ - + using System; using Newtonsoft.Json; @@ -20,6 +20,7 @@ public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, I [JsonProperty(Required = Required.Always)] public ExtractFileStatus Status { get; set; } + // TODO Consider renaming /// /// Anonymised file name. Only required if a file has been anonymised /// @@ -36,6 +37,9 @@ public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, I [JsonConstructor] public ExtractFileStatusMessage() { } + public ExtractFileStatusMessage(IExtractMessage request) + : base(request) { } + #region Equality Members diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 4feba0881..a050ddbbe 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -83,6 +83,7 @@ private GlobalOptions() { } public CohortPackagerOptions CohortPackagerOptions { get; set; } public DicomReprocessorOptions DicomReprocessorOptions { get; set; } public DicomTagReaderOptions DicomTagReaderOptions { get; set; } + public FileCopierOptions FileCopierOptions { get; set; } public IdentifierMapperOptions IdentifierMapperOptions { get; set; } public MongoDbPopulatorOptions MongoDbPopulatorOptions { get; set; } public ProcessDirectoryOptions ProcessDirectoryOptions { get; set; } @@ -286,6 +287,14 @@ public override string ToString() } } + [UsedImplicitly] + public class FileCopierOptions : ConsumerOptions + { + public ProducerOptions CopyStatusProducerOptions { get; set; } + + public override string ToString() => GlobalOptions.GenerateToString(this); + } + public enum TagProcessorMode { Serial, diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopier.cs new file mode 100644 index 000000000..908855a4a --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopier.cs @@ -0,0 +1,78 @@ +using JetBrains.Annotations; +using NLog; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using System.IO.Abstractions; + + +namespace Microservices.FileCopier.Execution +{ + public class FileCopier : IFileCopier + { + [NotNull] private readonly IProducerModel _copyStatusProducerModel; + + [NotNull] private readonly string _fileSystemRoot; + [NotNull] private readonly IFileSystem _fileSystem; + + [NotNull] private readonly ILogger _logger; + + + public FileCopier( + [NotNull] IProducerModel copyStatusCopyStatusProducerModel, + [NotNull] string fileSystemRoot, + [CanBeNull] IFileSystem fileSystem = null) + { + _copyStatusProducerModel = copyStatusCopyStatusProducerModel; + _fileSystemRoot = fileSystemRoot; + _fileSystem = fileSystem ?? new FileSystem(); + + _logger = LogManager.GetLogger(GetType().Name); + + } + + public void ProcessMessage( + [NotNull] ExtractFileMessage message, + [NotNull] IMessageHeader header) + { + string fullSrc = _fileSystem.Path.Join(_fileSystemRoot, message.DicomFilePath); + + ExtractFileStatusMessage statusMessage; + + if (!_fileSystem.File.Exists(fullSrc)) + { + statusMessage = new ExtractFileStatusMessage(message) + { + DicomFilePath = message.DicomFilePath, + Status = ExtractFileStatus.FileMissing, + StatusMessage = $"Could not find '{fullSrc}'" + }; + _copyStatusProducerModel.SendMessage(statusMessage, header); + return; + } + + string fullDest = _fileSystem.Path.Join(_fileSystemRoot, message.OutputPath); + + if (_fileSystem.File.Exists(fullDest)) + _logger.Warn($"Output file '{fullDest}' already exists. Will overwrite."); + + IDirectoryInfo parent = _fileSystem.Directory.GetParent(fullDest); + if (!parent.Exists) + { + _logger.Debug($"Creating directory '{parent}'"); + parent.Create(); + } + + _logger.Debug($"Copying source file to '{message.OutputPath}'"); + _fileSystem.File.Copy(fullSrc, fullDest, overwrite: true); + + statusMessage = new ExtractFileStatusMessage(message) + { + DicomFilePath = message.DicomFilePath, + Status = ExtractFileStatus.Copied, + AnonymisedFileName = message.OutputPath, + }; + _copyStatusProducerModel.SendMessage(statusMessage, header); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs new file mode 100644 index 000000000..388aaacec --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs @@ -0,0 +1,36 @@ +using JetBrains.Annotations; +using Microservices.FileCopier.Messaging; +using Smi.Common.Execution; +using Smi.Common.Messaging; +using Smi.Common.Options; +using System; +using System.IO; + +namespace Microservices.FileCopier.Execution +{ + public class FileCopierHost : MicroserviceHost + { + private readonly FileCopyQueueConsumer _consumer; + + public FileCopierHost( + [NotNull] GlobalOptions options, + bool loadSmiLogConfig = true) + : base(options, loadSmiLogConfig: loadSmiLogConfig) + { + if (!Directory.Exists(Globals.FileSystemOptions.FileSystemRoot)) + throw new ArgumentException($"Cannot find the specified FileSystemRoot: '{Globals.FileSystemOptions.FileSystemRoot}'"); + + Logger.Debug("Creating FileCopierHost with FileSystemRoot: " + Globals.FileSystemOptions.FileSystemRoot); + + IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false); + + var fileCopier = new FileCopier(copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); + _consumer = new FileCopyQueueConsumer(fileCopier); + } + + public override void Start() + { + RabbitMqAdapter.StartConsumer(Globals.FileCopierOptions, _consumer, isSolo: false); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs new file mode 100644 index 000000000..808ce30b7 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs @@ -0,0 +1,11 @@ +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; + + +namespace Microservices.FileCopier.Execution +{ + public interface IFileCopier + { + void ProcessMessage(ExtractFileMessage message, IMessageHeader header); + } +} diff --git a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs new file mode 100644 index 000000000..a8a2493e6 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs @@ -0,0 +1,39 @@ +using JetBrains.Annotations; +using Microservices.FileCopier.Execution; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using System; + +namespace Microservices.FileCopier.Messaging +{ + public class FileCopyQueueConsumer : Consumer + { + [NotNull] private readonly IFileCopier _fileCopier; + + public FileCopyQueueConsumer( + [NotNull] IFileCopier fileCopier) + { + _fileCopier = fileCopier; + } + + protected override void ProcessMessageImpl( + [NotNull] IMessageHeader header, + [NotNull] ExtractFileMessage message, + ulong tag) + { + try + { + _fileCopier.ProcessMessage(message, header); + } + catch (ApplicationException e) + { + // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage + ErrorAndNack(header, tag, "Error while processing ExtractFileStatusMessage", e); + return; + } + + Ack(header, tag); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Program.cs b/src/microservices/Microservices.FileCopier/Program.cs index 3adac6abc..46efe138b 100644 --- a/src/microservices/Microservices.FileCopier/Program.cs +++ b/src/microservices/Microservices.FileCopier/Program.cs @@ -1,7 +1,8 @@ using CommandLine; +using Microservices.FileCopier.Execution; +using Smi.Common.Execution; using Smi.Common.Options; - namespace Microservices.FileCopier { internal static class Program @@ -17,11 +18,10 @@ private static int Main(string[] args) { GlobalOptions options = GlobalOptions.Load(o); - //var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomTagReaderHost(options)); - //return bootstrapper.Main(); - return 0; + var bootstrapper = new MicroserviceHostBootstrapper(() => new FileCopierHost(options)); + return bootstrapper.Main(); }, - err => -100); + err => 1); } } } From b1e9d9fd80c583361214d9af025169eb287a1217 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 19:55:59 +0100 Subject: [PATCH 038/138] Add FileCopier test proj --- SmiServices.sln | 8 +++- .../Execution/FileCopierHostTest.cs | 37 +++++++++++++++++++ .../Execution/FileCopierTest.cs | 37 +++++++++++++++++++ .../Messaging/FileCopyQueueConsumerTest.cs | 37 +++++++++++++++++++ .../Microservices.FileCopier.Tests.csproj | 28 ++++++++++++++ 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs create mode 100644 tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs create mode 100644 tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs create mode 100644 tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj diff --git a/SmiServices.sln b/SmiServices.sln index 289fccd3a..7fa993746 100644 --- a/SmiServices.sln +++ b/SmiServices.sln @@ -75,8 +75,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiabl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IsIdentifiableReviewer", "src\applications\IsIdentifiableReviewer\IsIdentifiableReviewer.csproj", "{C2031E86-81B4-405A-A923-9B82E0CE196F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier.Tests", "tests\microservices\Microservices.FileCopier.Tests\Microservices.FileCopier.Tests.csproj", "{D61F6BF9-E857-457C-B745-40489A8CFE65}" Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -187,6 +188,10 @@ Global {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.Build.0 = Debug|x64 {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.ActiveCfg = Release|x64 {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.Build.0 = Release|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Debug|x64.ActiveCfg = Debug|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Debug|x64.Build.0 = Debug|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Release|x64.ActiveCfg = Release|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -218,6 +223,7 @@ Global {E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2} = {421CCD37-3817-4748-B184-A134E19DD75C} {C2031E86-81B4-405A-A923-9B82E0CE196F} = {8B943F2C-835B-484A-86D2-3F1462970605} {D4E52707-FFF7-41E6-8057-C6DB344B8CD7} = {421CCD37-3817-4748-B184-A134E19DD75C} + {D61F6BF9-E857-457C-B745-40489A8CFE65} = {421CCD37-3817-4748-B184-A134E19DD75C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {11CDEA53-71E8-4A9B-BC0D-74F4EB54F740} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs new file mode 100644 index 000000000..86af1135b --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -0,0 +1,37 @@ +using NUnit.Framework; + +namespace Microservices.FileCopier.Tests.Execution +{ + public class FileCopierHostTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() { } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void ExampleTest() + { + Assert.Fail(); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs new file mode 100644 index 000000000..a85e78c47 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -0,0 +1,37 @@ +using NUnit.Framework; + +namespace Microservices.FileCopier.Tests.Execution +{ + public class FileCopierTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() { } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void ExampleTest() + { + Assert.Fail(); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs new file mode 100644 index 000000000..726a32417 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -0,0 +1,37 @@ +using NUnit.Framework; + +namespace Microservices.FileCopier.Tests.Messaging +{ + public class FileCopyQueueConsumerTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() { } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void ExampleTest() + { + Assert.Fail(); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj new file mode 100644 index 000000000..bae8325a6 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj @@ -0,0 +1,28 @@ + + + Microservices.FileCopier.Tests + netcoreapp3.1 + false + bin\$(Platform)\$(Configuration)\ + x64 + false + false + true + 8.0 + full + true + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + From d6168fc1539ba0434c8bb88259e1ebd4c6db84b1 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 20:04:15 +0100 Subject: [PATCH 039/138] Update TODOs --- branch_todo.md | 4 +++- .../Execution/ExtractJobStorage/ExtractJobStore.cs | 2 ++ .../Messaging/AnonFailedMessageConsumer.cs | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/branch_todo.md b/branch_todo.md index aed9ba326..198ef672a 100644 --- a/branch_todo.md +++ b/branch_todo.md @@ -2,4 +2,6 @@ # Branch TODO - Update the extraction plan doc -- Ensure changes to message definitions are reflected in the Java code \ No newline at end of file +- Ensure changes to message definitions are reflected in the Java code +- Update RMQ exchange names, queue names and binding keys + - Java currently has them as "success" or "failure", which doesn't make sense anymore diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index ac498f88e..c6793d122 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -43,11 +43,13 @@ public void PersistMessageToStore( [NotNull] ExtractFileStatusMessage message, [NotNull] IMessageHeader header) { + // TODO if (message.Status == ExtractFileStatus.Unknown) throw new ApplicationException("ExtractFileStatus was unknown"); if (message.Status == ExtractFileStatus.Anonymised) throw new ApplicationException("Received an anonymisation successful message from the failure queue"); + // TODO PersistMessageToStoreImpl(message, header); } diff --git a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs index f5cbd910d..1431c419a 100644 --- a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs +++ b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs @@ -8,6 +8,7 @@ namespace Microservices.CohortPackager.Messaging { + // TODO Naming /// /// Consumer for (s) /// From e8c97303251c3f120457ab81fbbca12c786aea8f Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod <5160559+rkm@users.noreply.github.com> Date: Tue, 25 Aug 2020 22:43:59 +0100 Subject: [PATCH 040/138] Add initial tests for FileCopier --- branch_todo.md | 2 +- .../Messages/Extraction/ExtractFileMessage.cs | 2 +- .../Extraction/ExtractFileStatusMessage.cs | 7 + .../Smi.Common/Messages/MessagingConstants.cs | 8 + ...{FileCopier.cs => ExtractionFileCopier.cs} | 12 +- .../Execution/FileCopierHost.cs | 2 +- .../Messages/MessagingConstantsTest.cs | 43 ++++++ .../Execution/FileCopierHostTest.cs | 4 +- .../Execution/FileCopierTest.cs | 140 +++++++++++++++++- .../Messaging/FileCopyQueueConsumerTest.cs | 24 ++- 10 files changed, 226 insertions(+), 18 deletions(-) create mode 100644 src/common/Smi.Common/Messages/MessagingConstants.cs rename src/microservices/Microservices.FileCopier/Execution/{FileCopier.cs => ExtractionFileCopier.cs} (79%) create mode 100644 tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs diff --git a/branch_todo.md b/branch_todo.md index 198ef672a..74a94990f 100644 --- a/branch_todo.md +++ b/branch_todo.md @@ -4,4 +4,4 @@ - Update the extraction plan doc - Ensure changes to message definitions are reflected in the Java code - Update RMQ exchange names, queue names and binding keys - - Java currently has them as "success" or "failure", which doesn't make sense anymore + - Java currently has them as "success" or "failure", which doesn't make sense anymore. Change to "verify" and "noverify" diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs index 724a7db61..8dc6b91da 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs @@ -24,7 +24,7 @@ public class ExtractFileMessage : ExtractMessage, IFileReferenceMessage, IEquata [JsonConstructor] - private ExtractFileMessage() { } + public ExtractFileMessage() { } public ExtractFileMessage(ExtractionRequestMessage request) : base(request) { } diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs index c2468ed66..da665c302 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs @@ -40,6 +40,13 @@ public ExtractFileStatusMessage() { } public ExtractFileStatusMessage(IExtractMessage request) : base(request) { } + public override string ToString() => + $"{base.ToString()}," + + $"DicomFilePath={DicomFilePath}," + + $"ExtractFileStatus={Status}," + + $"AnonymisedFileName={AnonymisedFileName}," + + $"StatusMessage={StatusMessage}," + + ""; #region Equality Members diff --git a/src/common/Smi.Common/Messages/MessagingConstants.cs b/src/common/Smi.Common/Messages/MessagingConstants.cs new file mode 100644 index 000000000..900a1c0f6 --- /dev/null +++ b/src/common/Smi.Common/Messages/MessagingConstants.cs @@ -0,0 +1,8 @@ +namespace Smi.Common.Messages +{ + public static class MessagingConstants + { + public const string RMQ_EXTRACT_FILE_VERIFY_ROUTING_KEY = "verify"; + public const string RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY = "noverify"; + } +} diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs similarity index 79% rename from src/microservices/Microservices.FileCopier/Execution/FileCopier.cs rename to src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs index 908855a4a..b23f1ee47 100644 --- a/src/microservices/Microservices.FileCopier/Execution/FileCopier.cs +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -8,7 +8,7 @@ namespace Microservices.FileCopier.Execution { - public class FileCopier : IFileCopier + public class ExtractionFileCopier : IFileCopier { [NotNull] private readonly IProducerModel _copyStatusProducerModel; @@ -18,7 +18,7 @@ public class FileCopier : IFileCopier [NotNull] private readonly ILogger _logger; - public FileCopier( + public ExtractionFileCopier( [NotNull] IProducerModel copyStatusCopyStatusProducerModel, [NotNull] string fileSystemRoot, [CanBeNull] IFileSystem fileSystem = null) @@ -35,7 +35,7 @@ public void ProcessMessage( [NotNull] ExtractFileMessage message, [NotNull] IMessageHeader header) { - string fullSrc = _fileSystem.Path.Join(_fileSystemRoot, message.DicomFilePath); + string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath); ExtractFileStatusMessage statusMessage; @@ -47,11 +47,11 @@ public void ProcessMessage( Status = ExtractFileStatus.FileMissing, StatusMessage = $"Could not find '{fullSrc}'" }; - _copyStatusProducerModel.SendMessage(statusMessage, header); + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); return; } - string fullDest = _fileSystem.Path.Join(_fileSystemRoot, message.OutputPath); + string fullDest = _fileSystem.Path.Combine(_fileSystemRoot, message.ExtractionDirectory, message.OutputPath); if (_fileSystem.File.Exists(fullDest)) _logger.Warn($"Output file '{fullDest}' already exists. Will overwrite."); @@ -72,7 +72,7 @@ public void ProcessMessage( Status = ExtractFileStatus.Copied, AnonymisedFileName = message.OutputPath, }; - _copyStatusProducerModel.SendMessage(statusMessage, header); + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); } } } diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs index 388aaacec..9d74c887c 100644 --- a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs @@ -24,7 +24,7 @@ public FileCopierHost( IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false); - var fileCopier = new FileCopier(copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); + var fileCopier = new ExtractionFileCopier(copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); _consumer = new FileCopyQueueConsumer(fileCopier); } diff --git a/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs b/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs new file mode 100644 index 000000000..66939456d --- /dev/null +++ b/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs @@ -0,0 +1,43 @@ +using NUnit.Framework; +using Smi.Common.Messages; + + +namespace Smi.Common.Tests.Messages +{ + public class MessagingConstantsTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() { } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + /// + /// This test fails as a reminder that the associated RabbitMQ configuration needs to be updated! + /// + [Test] + public void Test_Constants_NotModified() + { + Assert.AreEqual("verify", MessagingConstants.RMQ_EXTRACT_FILE_VERIFY_ROUTING_KEY); + Assert.AreEqual("noverify", MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs index 86af1135b..49fceb8c2 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -27,9 +27,9 @@ public void TearDown() { } #region Tests [Test] - public void ExampleTest() + public void Test_FileCopierHost_HappyPath() { - Assert.Fail(); + Assert.Inconclusive(); } #endregion diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs index a85e78c47..d1fc9577b 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -1,13 +1,37 @@ -using NUnit.Framework; +using Microservices.FileCopier.Execution; +using Moq; +using NUnit.Framework; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using Smi.Common.Tests; +using System; +using System.IO.Abstractions.TestingHelpers; namespace Microservices.FileCopier.Tests.Execution { public class FileCopierTest { + private MockFileSystem _mockFileSystem; + private const string FileSystemRoot = "smi"; + private string _relativeSrc; + private readonly byte[] _expectedContents = { 0b00, 0b01, 0b10, 0b11 }; + private ExtractFileMessage _requestMessage; + #region Fixture Methods [OneTimeSetUp] - public void OneTimeSetUp() { } + public void OneTimeSetUp() + { + TestLogger.Setup(); + + _mockFileSystem = new MockFileSystem(); + _mockFileSystem.Directory.CreateDirectory(FileSystemRoot); + _relativeSrc = _mockFileSystem.Path.Combine("input", "a.dcm"); + string src = _mockFileSystem.Path.Combine("smi", _relativeSrc); + _mockFileSystem.Directory.CreateDirectory(_mockFileSystem.Directory.GetParent(src).FullName); + _mockFileSystem.File.WriteAllBytes(src, _expectedContents); + } [OneTimeTearDown] public void OneTimeTearDown() { } @@ -17,7 +41,18 @@ public void OneTimeTearDown() { } #region Test Methods [SetUp] - public void SetUp() { } + public void SetUp() + { + _requestMessage = new ExtractFileMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "123", + ExtractionDirectory = "extract", + DicomFilePath = _relativeSrc, + OutputPath = "out.dcm", + }; + } [TearDown] public void TearDown() { } @@ -27,9 +62,104 @@ public void TearDown() { } #region Tests [Test] - public void ExampleTest() + public void Test_FileCopier_HappyPath() + { + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + var requestHeader = new MessageHeader(); + + var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractFileStatus.Copied, + AnonymisedFileName = _requestMessage.OutputPath, + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + + string expectedDest = _mockFileSystem.Path.Combine("smi", "extract", "out.dcm"); + Assert.True(_mockFileSystem.File.Exists(expectedDest)); + Assert.AreEqual(_expectedContents, _mockFileSystem.File.ReadAllBytes(expectedDest)); + } + + [Test] + public void Test_FileCopier_MissingFile_SendsMessage() + { + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + _requestMessage.DicomFilePath = "missing.dcm"; + var requestHeader = new MessageHeader(); + + var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractFileStatus.FileMissing, + AnonymisedFileName = null, + StatusMessage = $"Could not find '{_mockFileSystem.Path.Combine(FileSystemRoot, "missing.dcm")}'" + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + } + + [Test] + public void Test_FileCopier_ExistingOutputFile_IsOverwritten() { - Assert.Fail(); + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + var requestHeader = new MessageHeader(); + string expectedDest = _mockFileSystem.Path.Combine("smi", "extract", "out.dcm"); + _mockFileSystem.Directory.GetParent(expectedDest).Create(); + _mockFileSystem.File.WriteAllBytes(expectedDest, new byte[] { 0b0 }); + + var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractFileStatus.Copied, + AnonymisedFileName = _requestMessage.OutputPath, + StatusMessage = null, + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + Assert.AreEqual(_expectedContents, _mockFileSystem.File.ReadAllBytes(expectedDest)); } #endregion diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs index 726a32417..9aa4f8e99 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -27,9 +27,29 @@ public void TearDown() { } #region Tests [Test] - public void ExampleTest() + public void Test_FileCopyQueueConsumer_ValidMessage_IsAcked() { - Assert.Fail(); + // There's a ridiculous amount of boilerplate required to test this at the moment... + //var mockFileCopier = new Mock(MockBehavior.Strict); + //var consumer = new FileCopyQueueConsumer(mockFileCopier.Object); + //consumer.ProcessMessage(); + + // TODO(rkm 2020-08-25) Test Ack / not Nack + Assert.Inconclusive(); + } + + [Test] + public void Test_FileCopyQueueConsumer_ApplicationException_IsNacked() + { + // TODO(rkm 2020-08-25) Test Nack / not Ack + Assert.Inconclusive(); + } + + [Test] + public void Test_FileCopyQueueConsumer_UnknownException_CallsFatalCallback() + { + // TODO(rkm 2020-08-25) Test not ack / not nack / Fatal called + Assert.Inconclusive(); } #endregion From f3b192d4315f355d7bf13b01ad25c2058aa6522d Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 25 Aug 2020 21:49:32 +0000 Subject: [PATCH 041/138] Fix SmiServices.sln VisualStudio and the dotnet CLI clearly have different opinions on what a valid sln file looks like... --- SmiServices.sln | 1 + 1 file changed, 1 insertion(+) diff --git a/SmiServices.sln b/SmiServices.sln index 7fa993746..dbb817ecf 100644 --- a/SmiServices.sln +++ b/SmiServices.sln @@ -78,6 +78,7 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier.Tests", "tests\microservices\Microservices.FileCopier.Tests\Microservices.FileCopier.Tests.csproj", "{D61F6BF9-E857-457C-B745-40489A8CFE65}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 From dcc7fcd529420d4c561f15469e452a855040d717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 05:01:49 +0000 Subject: [PATCH 042/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.6 to 3.5.7. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.6...v3.5.7) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 94d520473..294d4bba7 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.6 + 3.5.7 From d7d79415f61ce0b08c6b02727f8170f50dcda218 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 09:37:00 +0000 Subject: [PATCH 043/138] Bump HIC.RDMP.Dicom from 2.1.7 to 2.1.8 (#372) --- PACKAGES.md | 2 +- .../Microservices.DicomRelationalMapper.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index b3501bdc2..80d44405e 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -14,7 +14,7 @@ | fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.8](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index 4c8191e0d..855052aea 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -18,7 +18,7 @@ - + From c37cd3feb168dafb54904dfdbb2551c13cb60206 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 09:46:06 +0000 Subject: [PATCH 044/138] Bump MongoDB.Driver from 2.11.0 to 2.11.1 (#371) --- PACKAGES.md | 2 +- src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 80d44405e..eebaf3a25 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -20,7 +20,7 @@ | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | | Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.7](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL | -| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0/License) | For writting/reading dicom tags into MongoDb databases| +| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.1](https://www.nuget.org/packages/MongoDB.Driver/2.11.1)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.1/License) | For writting/reading dicom tags into MongoDb databases| | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | | RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | diff --git a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj index 73d0d104f..b86c5c947 100644 --- a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj +++ b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj @@ -14,7 +14,7 @@ - + From b35d1b543cdb1e8d31ad2ad673dddee2ba96cb9d Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 16:20:58 +0000 Subject: [PATCH 045/138] update RMQ extraction config --- data/rabbitmqConfigs/README.md | 9 ++++ .../rabbitmqConfigs/defaultExtractConfig.json | 51 ++++++++++++------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/data/rabbitmqConfigs/README.md b/data/rabbitmqConfigs/README.md index 445191edb..6155ae0bd 100644 --- a/data/rabbitmqConfigs/README.md +++ b/data/rabbitmqConfigs/README.md @@ -31,6 +31,15 @@ $ curl \ http://0.0.0.0:15672/api/definitions ``` +## Deleting a vhost + +```bash +> curl \ + -u guest:guest \ + -XDELETE \ + http://0.0.0.0:15672/api/vhosts/ +``` + ## Filter the default exchanges RabbitMQ has predefined exchanges which can't be removed from the management UI. To filter these out, tick the `regex` checkbox next to the search box on the `Exchanges` tab, then use this regex: diff --git a/data/rabbitmqConfigs/defaultExtractConfig.json b/data/rabbitmqConfigs/defaultExtractConfig.json index 80b2bbe9d..2453b7ed8 100644 --- a/data/rabbitmqConfigs/defaultExtractConfig.json +++ b/data/rabbitmqConfigs/defaultExtractConfig.json @@ -27,7 +27,7 @@ "arguments": {} }, { - "name": "AnonStatusExchange", + "name": "ExtractedFileStatusExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -36,7 +36,7 @@ "arguments": {} }, { - "name": "VerificationStatusExchange", + "name": "ExtractedFileVerifiedExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -54,7 +54,7 @@ "arguments": {} }, { - "name": "AnonFileExchange", + "name": "ExtractFileExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -115,21 +115,28 @@ "arguments": {} }, { - "name": "AnonSuccessQueue", + "name": "ExtractedFileToVerifyQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, "arguments": {} }, { - "name": "AnonFailedQueue", + "name": "ExtractedFileNoVerifyQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, "arguments": {} }, { - "name": "AnonFileQueue", + "name": "ExtractFileAnonQueue", + "vhost": "smi_extract", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "ExtractFileIdentQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, @@ -150,7 +157,7 @@ "arguments": {} }, { - "name": "VerificationStatusQueue", + "name": "ExtractedFileVerifiedQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, @@ -182,27 +189,35 @@ "arguments": {} }, { - "source": "AnonFileExchange", + "source": "ExtractFileExchange", "vhost": "smi_extract", - "destination": "AnonFileQueue", + "destination": "ExtractFileAnonQueue", "destination_type": "queue", - "routing_key": "", + "routing_key": "anon", + "arguments": {} + }, + { + "source": "ExtractFileExchange", + "vhost": "smi_extract", + "destination": "ExtractFileIdentQueue", + "destination_type": "queue", + "routing_key": "ident", "arguments": {} }, { - "source": "AnonStatusExchange", + "source": "ExtractedFileStatusExchange", "vhost": "smi_extract", - "destination": "AnonSuccessQueue", + "destination": "ExtractedFileToVerifyQueue", "destination_type": "queue", - "routing_key": "success", + "routing_key": "verify", "arguments": {} }, { - "source": "AnonStatusExchange", + "source": "ExtractedFileStatusExchange", "vhost": "smi_extract", - "destination": "AnonFailedQueue", + "destination": "ExtractedFileNoVerifyQueue", "destination_type": "queue", - "routing_key": "failure", + "routing_key": "noverify", "arguments": {} }, { @@ -222,9 +237,9 @@ "arguments": {} }, { - "source": "VerificationStatusExchange", + "source": "ExtractedFileVerifiedExchange", "vhost": "smi_extract", - "destination": "VerificationStatusQueue", + "destination": "ExtractedFileVerifiedQueue", "destination_type": "queue", "routing_key": "", "arguments": {} From cfada0f524a9d90b5188885ddc1355ae48575888 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 16:29:20 +0000 Subject: [PATCH 046/138] update default config and queue names --- data/microserviceConfigs/default.yaml | 23 +++++++++++++------ .../Smi.Common/Options/GlobalOptions.cs | 2 +- src/common/Smi_Common_Python/Rabbit.py | 2 +- .../org/smi/common/options/GlobalOptions.java | 2 +- .../Execution/CohortPackagerHost.cs | 2 +- .../README.md | 2 +- .../execution/CTPAnonymiserHost.java | 2 +- .../test/execution/CTPAnonymiserHostTest.java | 8 +++---- .../GlobalOptionsExtensions.cs | 2 +- .../Execution/CohortPackagerHostTest.cs | 4 ++-- 10 files changed, 29 insertions(+), 20 deletions(-) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index 50bd36646..3d3620f9d 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -93,12 +93,12 @@ CohortPackagerOptions: QueueName: 'TEST.FileCollectionInfoQueue' QoSPrefetchCount: 1 AutoAck: false - AnonFailedOptions: - QueueName: 'TEST.AnonFailedQueue' + NoVerifyStatusOptions: + QueueName: 'TEST.ExtractedFileNoVerifyQueue' QoSPrefetchCount: 1 AutoAck: false VerificationStatusOptions: - QueueName: 'TEST.VerificationStatusQueue' + QueueName: 'TEST.ExtractedFileVerifiedQueue' QoSPrefetchCount: 1 AutoAck: false @@ -158,14 +158,23 @@ ProcessDirectoryOptions: MaxConfirmAttempts: 1 CTPAnonymiserOptions: - ExtractFileConsumerOptions: - QueueName: 'TEST.ExtractFileQueue' + AnonFileConsumerOptions: + QueueName: 'TEST.ExtractFileAnonQueue' QoSPrefetchCount: 1 AutoAck: false ExtractFileStatusProducerOptions: ExchangeName: 'TEST.FileStatusExchange' MaxConfirmAttempts: 1 +FileCopierOptions: + CopyFileConsumerOptions: + QueueName: 'TEST.ExtractFileIdentQueue' + QoSPrefetchCount: 1 + AutoAck: false + CopyStatusProducerOptions: + ExchangeName: 'TEST.FileStatusExchange' + MaxConfirmAttempts: 1 + ExtractorClOptions: MaxIdentifiersPerMessage: 1000 ExtractionRequestProducerOptions: @@ -176,11 +185,11 @@ ExtractorClOptions: MaxConfirmAttempts: 1 IsIdentifiableOptions: - QueueName: 'TEST.IsIdentifiableQueue' + QueueName: 'TEST.ExtractedFileToVerifyQueue' QoSPrefetchCount: 1 AutoAck: false IsIdentifiableProducerOptions: - ExchangeName: 'TEST.IsIdentifiableExchange' + ExchangeName: 'TEST.ExtractedFileVerifiedExchange' MaxConfirmAttempts: 1 ClassifierType: 'Microservices.IsIdentifiable.Service.TesseractStanfordDicomFileClassifier' DataDirectory: '' diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index a050ddbbe..3acf803fb 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -339,7 +339,7 @@ public class CohortPackagerOptions { public ConsumerOptions ExtractRequestInfoOptions { get; set; } public ConsumerOptions FileCollectionInfoOptions { get; set; } - public ConsumerOptions AnonFailedOptions { get; set; } + public ConsumerOptions NoVerifyStatusOptions { get; set; } public ConsumerOptions VerificationStatusOptions { get; set; } public uint JobWatcherTimeoutInSeconds { get; set; } public string ReporterType { get; set; } diff --git a/src/common/Smi_Common_Python/Rabbit.py b/src/common/Smi_Common_Python/Rabbit.py index bc20a61ec..cf04fa85f 100755 --- a/src/common/Smi_Common_Python/Rabbit.py +++ b/src/common/Smi_Common_Python/Rabbit.py @@ -113,7 +113,7 @@ def send_CTP_Start_Message(yaml_dict, input_file, extraction_dir, project_name): """ Sends a Message to exchange given by QueueName requesting that CTP anonymises a DICOM file. NOTE! Sends a message direct to CTP input queue, not to the exchange used by CohortExtractor. """ - queueName = yaml_dict['CTPAnonymiserOptions']['ExtractFileConsumerOptions']['QueueName'] + queueName = yaml_dict['CTPAnonymiserOptions']['AnonFileConsumerOptions']['QueueName'] pika_connection = pika.BlockingConnection(get_pika_connection_parameters(yaml_dict)) pika_model = pika_connection.channel() diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java index 97e70f056..8cae3be45 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java @@ -130,7 +130,7 @@ public String getExtractRoot() { public class CTPAnonymiserOptions { - public ConsumerOptions ExtractFileConsumerOptions; + public ConsumerOptions AnonFileConsumerOptions; public ProducerOptions ExtractFileStatusProducerOptions; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs b/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs index 231a05543..39948e677 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs @@ -75,7 +75,7 @@ public override void Start() // TODO(rkm 2020-03-02) Once this is transactional, we can have one "master" service which actually does the job checking RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.ExtractRequestInfoOptions, _requestInfoMessageConsumer, isSolo: true); RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.FileCollectionInfoOptions, _fileCollectionMessageConsumer, isSolo: true); - RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.AnonFailedOptions, _anonFailedMessageConsumer, isSolo: true); + RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.NoVerifyStatusOptions, _anonFailedMessageConsumer, isSolo: true); RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.VerificationStatusOptions, _anonVerificationMessageConsumer, isSolo: true); } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/README.md b/src/microservices/com.smi.microservices.ctpanonymiser/README.md index 07b4a3044..efcbd6c7c 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/README.md +++ b/src/microservices/com.smi.microservices.ctpanonymiser/README.md @@ -21,7 +21,7 @@ The anonymiser is installed via Maven as per the other Java apps, so clone the p | Read/Write | Type | Config setting | | ------------- | ------------- |------------- | -| Read| ExtractFileMessage | `CTPAnonymiserOptions.ExtractFileConsumerOptions` | +| Read| ExtractFileMessage | `CTPAnonymiserOptions.AnonFileConsumerOptions` | | Write| ExtractFileStatusMessage|`CTPAnonymiserOptions.ExtractFileStatusProducerOptions`| The ExtractFileMessage contents include the name of a directory of DICOM files to be anonymised. As the files are anonymised, ExtractFileStatusMessage messages are produced indicating success or otherwise. diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java index 61b3eea81..fd3338af4 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java @@ -71,7 +71,7 @@ public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws I _logger.info("CTPAnonymiserHost created successfully"); // Start the consumer - _rabbitMqAdapter.StartConsumer(_options.CTPAnonymiserOptions.ExtractFileConsumerOptions, _consumer); + _rabbitMqAdapter.StartConsumer(_options.CTPAnonymiserOptions.AnonFileConsumerOptions, _consumer); } public IProducerModel getProducer() { diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index 6c36016f6..9e78aaa0e 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -67,15 +67,15 @@ protected void setUp() throws Exception { _options.FileSystemOptions.setFileSystemRoot(_fsRoot); _options.FileSystemOptions.setExtractRoot(_extractRoot); - if (!_options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName.startsWith("TEST.")) - _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName = "TEST." - + _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + if (!_options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName.startsWith("TEST.")) + _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName = "TEST." + + _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName; if (!_options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName.startsWith("TEST.")) _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName = "TEST." + _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; - String _consumerQueueName = _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + String _consumerQueueName = _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName; _producerExchangeName = _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; // Set up RMQ diff --git a/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs b/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs index 000e75829..7fa66f323 100644 --- a/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs +++ b/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs @@ -60,7 +60,7 @@ public static void UseTestValues(this GlobalOptions g, ConnectionFactory rabbit, g.CohortExtractorOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.ExtractRequestInfoOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.FileCollectionInfoOptions.QoSPrefetchCount = 1; - g.CohortPackagerOptions.AnonFailedOptions.QoSPrefetchCount = 1; + g.CohortPackagerOptions.NoVerifyStatusOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.VerificationStatusOptions.QoSPrefetchCount = 1; g.DicomTagReaderOptions.QoSPrefetchCount = 1; g.IdentifierMapperOptions.QoSPrefetchCount = 1; diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs index dde9480b2..1f982a1ad 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs @@ -129,12 +129,12 @@ public void TestCohortPackagerHost_HappyPath() globals.RabbitOptions, globals.CohortPackagerOptions.ExtractRequestInfoOptions, globals.CohortPackagerOptions.FileCollectionInfoOptions, - globals.CohortPackagerOptions.AnonFailedOptions, + globals.CohortPackagerOptions.NoVerifyStatusOptions, globals.CohortPackagerOptions.VerificationStatusOptions)) { tester.SendMessage(globals.CohortPackagerOptions.ExtractRequestInfoOptions, new MessageHeader(), testExtractionRequestInfoMessage); tester.SendMessage(globals.CohortPackagerOptions.FileCollectionInfoOptions, new MessageHeader(), testExtractFileCollectionInfoMessage); - tester.SendMessage(globals.CohortPackagerOptions.AnonFailedOptions, new MessageHeader(), testExtractFileStatusMessage); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage); tester.SendMessage(globals.CohortPackagerOptions.VerificationStatusOptions, new MessageHeader(), testIsIdentifiableMessage); var reporter = new TestReporter(); From 7b11aac63aad000309ab197a1bff2ef16c0d17c2 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 17:54:10 +0100 Subject: [PATCH 047/138] Make routing keys configurable via yaml --- data/microserviceConfigs/default.yaml | 5 +++ .../Smi.Common/Messages/MessagingConstants.cs | 8 ---- .../Smi.Common/Options/GlobalOptions.cs | 4 ++ .../Execution/ExtractionFileCopier.cs | 9 +++- .../Execution/FileCopierHost.cs | 2 +- .../Messages/MessagingConstantsTest.cs | 43 ------------------- .../Execution/FileCopierTest.cs | 20 ++++++--- 7 files changed, 31 insertions(+), 60 deletions(-) delete mode 100644 src/common/Smi.Common/Messages/MessagingConstants.cs delete mode 100644 tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index 3d3620f9d..dad2888d8 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -72,6 +72,8 @@ CohortExtractorOptions: AuditorType: 'Microservices.CohortExtractor.Audit.NullAuditExtractions' RequestFulfillerType: 'Microservices.CohortExtractor.Execution.RequestFulfillers.FromCataloguesExtractionRequestFulfiller' ProjectPathResolverType: 'Microservices.CohortExtractor.Execution.ProjectPathResolvers.DefaultProjectPathResolver' + ExtractAnonRoutingKey: anon + ExtractIdentRoutingKey: ident # Writes (Producer) to this exchange ExtractFilesProducerOptions: ExchangeName: 'TEST.ExtractFileExchange' @@ -158,6 +160,8 @@ ProcessDirectoryOptions: MaxConfirmAttempts: 1 CTPAnonymiserOptions: + VerifyRoutingKey: verify + NoVerifyRoutingKey: noverify AnonFileConsumerOptions: QueueName: 'TEST.ExtractFileAnonQueue' QoSPrefetchCount: 1 @@ -167,6 +171,7 @@ CTPAnonymiserOptions: MaxConfirmAttempts: 1 FileCopierOptions: + NoVerifyRoutingKey: noverify CopyFileConsumerOptions: QueueName: 'TEST.ExtractFileIdentQueue' QoSPrefetchCount: 1 diff --git a/src/common/Smi.Common/Messages/MessagingConstants.cs b/src/common/Smi.Common/Messages/MessagingConstants.cs deleted file mode 100644 index 900a1c0f6..000000000 --- a/src/common/Smi.Common/Messages/MessagingConstants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Smi.Common.Messages -{ - public static class MessagingConstants - { - public const string RMQ_EXTRACT_FILE_VERIFY_ROUTING_KEY = "verify"; - public const string RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY = "noverify"; - } -} diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 3acf803fb..2fcc49a31 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -291,6 +291,7 @@ public override string ToString() public class FileCopierOptions : ConsumerOptions { public ProducerOptions CopyStatusProducerOptions { get; set; } + public string NoVerifyRoutingKey { get; set; } public override string ToString() => GlobalOptions.GenerateToString(this); } @@ -397,6 +398,9 @@ public string AuditorType /// public List Blacklists { get; set; } + public string ExtractAnonRoutingKey { get; set; } + public string ExtractIdentRoutingKey { get; set; } + public ProducerOptions ExtractFilesProducerOptions { get; set; } public ProducerOptions ExtractFilesInfoProducerOptions { get; set; } diff --git a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs index b23f1ee47..0342fba75 100644 --- a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -3,6 +3,7 @@ using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; +using Smi.Common.Options; using System.IO.Abstractions; @@ -10,6 +11,8 @@ namespace Microservices.FileCopier.Execution { public class ExtractionFileCopier : IFileCopier { + [NotNull] private readonly FileCopierOptions _options; + [NotNull] private readonly IProducerModel _copyStatusProducerModel; [NotNull] private readonly string _fileSystemRoot; @@ -19,10 +22,12 @@ public class ExtractionFileCopier : IFileCopier public ExtractionFileCopier( + [NotNull] FileCopierOptions options, [NotNull] IProducerModel copyStatusCopyStatusProducerModel, [NotNull] string fileSystemRoot, [CanBeNull] IFileSystem fileSystem = null) { + _options = options; _copyStatusProducerModel = copyStatusCopyStatusProducerModel; _fileSystemRoot = fileSystemRoot; _fileSystem = fileSystem ?? new FileSystem(); @@ -47,7 +52,7 @@ public void ProcessMessage( Status = ExtractFileStatus.FileMissing, StatusMessage = $"Could not find '{fullSrc}'" }; - _ = _copyStatusProducerModel.SendMessage(statusMessage, header, MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); return; } @@ -72,7 +77,7 @@ public void ProcessMessage( Status = ExtractFileStatus.Copied, AnonymisedFileName = message.OutputPath, }; - _ = _copyStatusProducerModel.SendMessage(statusMessage, header, MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); } } } diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs index 9d74c887c..5eee5b62e 100644 --- a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs @@ -24,7 +24,7 @@ public FileCopierHost( IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false); - var fileCopier = new ExtractionFileCopier(copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); + var fileCopier = new ExtractionFileCopier(Globals.FileCopierOptions, copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); _consumer = new FileCopyQueueConsumer(fileCopier); } diff --git a/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs b/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs deleted file mode 100644 index 66939456d..000000000 --- a/tests/common/Smi.Common.Tests/Messages/MessagingConstantsTest.cs +++ /dev/null @@ -1,43 +0,0 @@ -using NUnit.Framework; -using Smi.Common.Messages; - - -namespace Smi.Common.Tests.Messages -{ - public class MessagingConstantsTest - { - #region Fixture Methods - - [OneTimeSetUp] - public void OneTimeSetUp() { } - - [OneTimeTearDown] - public void OneTimeTearDown() { } - - #endregion - - #region Test Methods - - [SetUp] - public void SetUp() { } - - [TearDown] - public void TearDown() { } - - #endregion - - #region Tests - - /// - /// This test fails as a reminder that the associated RabbitMQ configuration needs to be updated! - /// - [Test] - public void Test_Constants_NotModified() - { - Assert.AreEqual("verify", MessagingConstants.RMQ_EXTRACT_FILE_VERIFY_ROUTING_KEY); - Assert.AreEqual("noverify", MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY); - } - - #endregion - } -} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs index d1fc9577b..ebe4fdef8 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -4,6 +4,7 @@ using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; +using Smi.Common.Options; using Smi.Common.Tests; using System; using System.IO.Abstractions.TestingHelpers; @@ -12,6 +13,8 @@ namespace Microservices.FileCopier.Tests.Execution { public class FileCopierTest { + private FileCopierOptions _options; + private MockFileSystem _mockFileSystem; private const string FileSystemRoot = "smi"; private string _relativeSrc; @@ -25,6 +28,11 @@ public void OneTimeSetUp() { TestLogger.Setup(); + _options = new FileCopierOptions + { + NoVerifyRoutingKey = "noverify", + }; + _mockFileSystem = new MockFileSystem(); _mockFileSystem.Directory.CreateDirectory(FileSystemRoot); _relativeSrc = _mockFileSystem.Path.Combine("input", "a.dcm"); @@ -78,7 +86,7 @@ public void Test_FileCopier_HappyPath() var requestHeader = new MessageHeader(); - var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) @@ -88,7 +96,7 @@ public void Test_FileCopier_HappyPath() AnonymisedFileName = _requestMessage.OutputPath, }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); - Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); string expectedDest = _mockFileSystem.Path.Combine("smi", "extract", "out.dcm"); Assert.True(_mockFileSystem.File.Exists(expectedDest)); @@ -113,7 +121,7 @@ public void Test_FileCopier_MissingFile_SendsMessage() _requestMessage.DicomFilePath = "missing.dcm"; var requestHeader = new MessageHeader(); - var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) @@ -124,7 +132,7 @@ public void Test_FileCopier_MissingFile_SendsMessage() StatusMessage = $"Could not find '{_mockFileSystem.Path.Combine(FileSystemRoot, "missing.dcm")}'" }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); - Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); } [Test] @@ -147,7 +155,7 @@ public void Test_FileCopier_ExistingOutputFile_IsOverwritten() _mockFileSystem.Directory.GetParent(expectedDest).Create(); _mockFileSystem.File.WriteAllBytes(expectedDest, new byte[] { 0b0 }); - var copier = new ExtractionFileCopier(mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) @@ -158,7 +166,7 @@ public void Test_FileCopier_ExistingOutputFile_IsOverwritten() StatusMessage = null, }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); - Assert.AreEqual(MessagingConstants.RMQ_EXTRACT_FILE_NOVERIFY_ROUTING_KEY, sentRoutingKey); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); Assert.AreEqual(_expectedContents, _mockFileSystem.File.ReadAllBytes(expectedDest)); } From b17a81e616792f28f15d7f9901514da58879a590 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 18:19:07 +0100 Subject: [PATCH 048/138] Add IsIdentifiableExtraction to CohortExtractor --- .../Messages/Extraction/ExtractMessage.cs | 26 ++++--- .../Messages/Extraction/IExtractMessage.cs | 5 ++ .../Execution/CohortExtractorHost.cs | 2 +- .../ExtractionRequestQueueConsumer.cs | 15 ++-- .../Smi.Common.Tests/MessageEqualityTests.cs | 33 +++++++++ .../ExtractionRequestQueueConsumerTest.cs | 70 +++++++++++++++++++ 6 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs index dd0dd659e..89512aac0 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs @@ -21,17 +21,21 @@ public abstract class ExtractMessage : IExtractMessage, IEquatable $"ExtractionJobIdentifier={ExtractionJobIdentifier}, " + - $"ProjectNumber={ProjectNumber}, " + - $"ExtractionDirectory={ExtractionDirectory}, " + - $"JobSubmittedAt={JobSubmittedAt}"; + public override string ToString() => + $"ExtractionJobIdentifier={ExtractionJobIdentifier}, " + + $"ProjectNumber={ProjectNumber}, " + + $"ExtractionDirectory={ExtractionDirectory}, " + + $"JobSubmittedAt={JobSubmittedAt}, " + + $"IsIdentifiableExtraction={IsIdentifiableExtraction}, " + + ""; #region Equality Members @@ -75,9 +82,10 @@ public override int GetHashCode() unchecked { int hashCode = ExtractionJobIdentifier.GetHashCode(); - hashCode = (hashCode * 397) ^ (ProjectNumber != null ? ProjectNumber.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (ExtractionDirectory != null ? ExtractionDirectory.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); + hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode(); + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); return hashCode; } } diff --git a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs index 6bf2771ec..6ee3b5379 100644 --- a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs @@ -30,5 +30,10 @@ public interface IExtractMessage : IMessage /// DateTime the job was submitted at /// DateTime JobSubmittedAt { get; set; } + + /// + /// True if this is an identifiable extraction (i.e. files should not be anonymised) + /// + bool IsIdentifiableExtraction { get; } } } diff --git a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs index e01abcf08..c927390f6 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs @@ -78,7 +78,7 @@ public override void Start() InitializeExtractionSources(repositoryLocator); - Consumer = new ExtractionRequestQueueConsumer(_fulfiller, _auditor, _pathResolver, _fileMessageProducer, fileMessageInfoProducer); + Consumer = new ExtractionRequestQueueConsumer(Globals.CohortExtractorOptions, _fulfiller, _auditor, _pathResolver, _fileMessageProducer, fileMessageInfoProducer); RabbitMqAdapter.StartConsumer(_consumerOptions, Consumer, isSolo: false); } diff --git a/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs b/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs index 5809d09c0..8bd11e295 100644 --- a/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs +++ b/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs @@ -1,18 +1,19 @@ - -using System.ComponentModel; using Microservices.CohortExtractor.Audit; using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.ProjectPathResolvers; using Microservices.CohortExtractor.Execution.RequestFulfillers; -using RabbitMQ.Client.Events; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; +using Smi.Common.Options; +using System.ComponentModel; namespace Microservices.CohortExtractor.Messaging { public class ExtractionRequestQueueConsumer : Consumer { + private readonly CohortExtractorOptions _options; + private readonly IExtractionRequestFulfiller _fulfiller; private readonly IAuditExtractions _auditor; private readonly IProducerModel _fileMessageProducer; @@ -20,10 +21,13 @@ public class ExtractionRequestQueueConsumer : Consumer private readonly IProjectPathResolver _resolver; - public ExtractionRequestQueueConsumer(IExtractionRequestFulfiller fulfiller, IAuditExtractions auditor, + public ExtractionRequestQueueConsumer( + CohortExtractorOptions options, + IExtractionRequestFulfiller fulfiller, IAuditExtractions auditor, IProjectPathResolver pathResolver, IProducerModel fileMessageProducer, IProducerModel fileMessageInfoProducer) { + _options = options; _fulfiller = fulfiller; _auditor = auditor; _resolver = pathResolver; @@ -44,6 +48,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequ } string extractionDirectory = request.ExtractionDirectory.TrimEnd('/', '\\'); + string extractFileRoutingKey = request.IsIdentifiableExtraction ? _options.ExtractIdentRoutingKey : _options.ExtractAnonRoutingKey; foreach (ExtractImageCollection matchedFiles in _fulfiller.GetAllMatchingFiles(request, _auditor)) { @@ -66,7 +71,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequ Logger.Debug($"DicomFilePath={extractFileMessage.DicomFilePath}, OutputPath={extractFileMessage.OutputPath}"); // Send the extract file message - var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header); + var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header, extractFileRoutingKey); // Record that we sent it infoMessage.ExtractFileMessagesDispatched.Add(sentHeader, extractFileMessage.OutputPath); diff --git a/tests/common/Smi.Common.Tests/MessageEqualityTests.cs b/tests/common/Smi.Common.Tests/MessageEqualityTests.cs index b962fd535..a462e1b3d 100644 --- a/tests/common/Smi.Common.Tests/MessageEqualityTests.cs +++ b/tests/common/Smi.Common.Tests/MessageEqualityTests.cs @@ -1,9 +1,11 @@  using NUnit.Framework; using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; using System; using System.Linq; + namespace Smi.Common.Tests { public class MessageEqualityTests @@ -112,5 +114,36 @@ public void TestEquals_SeriesMessage() Assert.AreEqual(msg1, msg2); Assert.AreEqual(msg1.GetHashCode(), msg2.GetHashCode()); } + + + private class FooExtractMessage : ExtractMessage { } + + [Test] + public void Tests_ExtractMessage_Equality() + { + Guid guid = Guid.NewGuid(); + DateTime dt = DateTime.UtcNow; + + // TODO(rkm 2020-08-26) Swap these object initializers for proper constructors + var msg1 = new FooExtractMessage + { + JobSubmittedAt = dt, + ExtractionJobIdentifier = guid, + ProjectNumber = "1234", + ExtractionDirectory = "foo/bar", + IsIdentifiableExtraction = true, + }; + var msg2 = new FooExtractMessage + { + JobSubmittedAt = dt, + ExtractionJobIdentifier = guid, + ProjectNumber = "1234", + ExtractionDirectory = "foo/bar", + IsIdentifiableExtraction = true, + }; + + Assert.AreEqual(msg1, msg2); + Assert.AreEqual(msg1.GetHashCode(), msg2.GetHashCode()); + } } } diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs new file mode 100644 index 000000000..0b7e6f46a --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -0,0 +1,70 @@ +using Microservices.CohortExtractor.Audit; +using Microservices.CohortExtractor.Execution.ProjectPathResolvers; +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Microservices.CohortExtractor.Messaging; +using Moq; +using NUnit.Framework; +using Smi.Common.Messaging; +using Smi.Common.Options; + + +namespace Microservices.CohortExtractor.Tests.Messaging +{ + public class ExtractionRequestQueueConsumerTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() { } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() + { + var options = new CohortExtractorOptions(); + + var mockFulfiller = new Mock(MockBehavior.Strict); + var mockAuditor = new Mock(MockBehavior.Strict); + var mockPathResolver = new Mock(MockBehavior.Strict); + + var mockFileMessageProducerModel = new Mock(MockBehavior.Strict); + var mockFileInfoMessageProducerModel = new Mock(MockBehavior.Strict); + + var consumer = new ExtractionRequestQueueConsumer( + options, + mockFulfiller.Object, + mockAuditor.Object, mockPathResolver.Object, + mockFileMessageProducerModel.Object, + mockFileInfoMessageProducerModel.Object); + + // TODO + //consumer.ProcessMessage(); + + Assert.Inconclusive(); + } + + [Test] + public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() + { + Assert.Inconclusive(); + } + + #endregion + } +} From 7aa46aea6d8b7411dd31287be888c9b35f3e5612 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 22:31:52 +0100 Subject: [PATCH 049/138] Add new routing keys to CTP --- .../org/smi/common/options/GlobalOptions.java | 3 ++- .../execution/CTPAnonymiserHost.java | 3 ++- .../messaging/CTPAnonymiserConsumer.java | 16 ++++++++++++---- .../test/messages/ExtractFileMessageTest.java | 6 ++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java index 8cae3be45..7adfe313e 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java @@ -129,7 +129,8 @@ public String getExtractRoot() { } public class CTPAnonymiserOptions { - + public String VerifyRoutingKey; + public String NoVerifyRoutingKey; public ConsumerOptions AnonFileConsumerOptions; public ProducerOptions ExtractFileStatusProducerOptions; } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java index fd3338af4..9c1dc1ccc 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java @@ -23,7 +23,7 @@ public class CTPAnonymiserHost implements IMicroserviceHost { private final CTPAnonymiserConsumer _consumer; private IProducerModel _producer; - private final GlobalOptions _options; +private final GlobalOptions _options; public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws IOException, TimeoutException { @@ -63,6 +63,7 @@ public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws I SmiCtpProcessor anonTool = new DicomAnonymizerToolBuilder().tagAnonScriptFile(anonScriptFile).check(null).buildDat(); _consumer = new CTPAnonymiserConsumer( + _options, _producer, anonTool, fsRoot, diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java index c49b46a49..0ae68590b 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java @@ -13,6 +13,7 @@ import org.smi.ctpanonymiser.messages.ExtractFileMessage; import org.smi.ctpanonymiser.messages.ExtractFileStatusMessage; import org.smi.ctpanonymiser.util.CtpAnonymisationStatus; +import org.smi.common.options.GlobalOptions; import org.smi.ctpanonymiser.util.ExtractFileStatus; import java.io.File; @@ -23,8 +24,12 @@ public class CTPAnonymiserConsumer extends SmiConsumer { private final static Logger _logger = Logger.getRootLogger(); - private final static String _routingKey_failure = "failure"; - private final static String _routingKey_success = "success"; + + private GlobalOptions _options; + + private String _routingKey_failure; + private String _routingKey_success; + private String _fileSystemRoot; private String _extractFileSystemRoot; @@ -35,9 +40,12 @@ public class CTPAnonymiserConsumer extends SmiConsumer { private boolean _foundAFile = false; - public CTPAnonymiserConsumer(IProducerModel producer, SmiCtpProcessor anonTool, String fileSystemRoot, + public CTPAnonymiserConsumer(GlobalOptions options, IProducerModel producer, SmiCtpProcessor anonTool, String fileSystemRoot, String extractFileSystemRoot) { + _routingKey_failure = options.CTPAnonymiserOptions.NoVerifyRoutingKey; + _routingKey_success = options.CTPAnonymiserOptions.VerifyRoutingKey; + _statusMessageProducer = producer; _anonTool = anonTool; _fileSystemRoot = fileSystemRoot; @@ -136,7 +144,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope if (!tempFile.delete() || tempFile.exists()) _logger.warn("Could not delete temp file " + tempFile.getAbsolutePath()); - String routingKey = _routingKey_failure; + String routingKey; if (status == CtpAnonymisationStatus.Anonymised) { diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java index ef47b81bc..cf3e4ce62 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java @@ -6,6 +6,7 @@ import org.apache.log4j.Logger; import org.smi.common.logging.SmiLogging; +import org.smi.common.options.GlobalOptions; import org.smi.ctpanonymiser.messages.ExtractFileMessage; import org.smi.ctpanonymiser.messaging.CTPAnonymiserConsumer; @@ -44,10 +45,11 @@ protected void tearDown() throws Exception { super.tearDown(); } - public void testSerializeDeserialize() { + public void testSerializeDeserialize() throws Exception { ExtractFileMessage recvdMessage; - CTPAnonymiserConsumer consumer = new CTPAnonymiserConsumer(null, null, _fileSystemRoot, _extractFileSystemRoot); + GlobalOptions options = GlobalOptions.Load(true); + CTPAnonymiserConsumer consumer = new CTPAnonymiserConsumer(options, null, null, _fileSystemRoot, _extractFileSystemRoot); // Get byte array version of message Gson _gson = new Gson(); From 1d20cf3b4b7aee968ad2c730ff96528663d4d691 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 22:32:30 +0100 Subject: [PATCH 050/138] Make Java ExtractFileStatus match c# New fields are unused in java-land --- .../smi/ctpanonymiser/util/ExtractFileStatus.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java index c87f05bf1..ade2389ec 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java @@ -18,5 +18,15 @@ public enum ExtractFileStatus { /** * The file could not be anonymised and will not be retired */ - ErrorWontRetry; + ErrorWontRetry, + + /** + * The source file could not be found under the given filesystem root + */ + FileMissing, + + /** + * The source file was successfully copied to the destination + */ + Copied, } From 9946576ceed6cc5f65bdede83b2569e0263abf69 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 22:46:59 +0100 Subject: [PATCH 051/138] Add IsIdentifiableExtraction to extractorCli --- .../src/main/java/org/smi/extractorcl/Program.java | 9 +++++++++ .../org/smi/extractorcl/execution/ExtractorClHost.java | 3 +++ .../extractorcl/fileUtils/ExtractMessagesCsvHandler.java | 7 ++++++- .../java/org/smi/common/messages/ExtractMessage.java | 6 ++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java index a488e9b0f..74dd579c8 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java @@ -88,6 +88,15 @@ private static CommandLine ParseOptions(String[] args) throws ParseException { .hasArg() .longOpt("modality") .build()); + + options.addOption( + Option + .builder("i") + .type(boolean.class) + .argName("identifiable extraction") + .desc("This is an identifiable extraction") + .longOpt("identifiable-extraction") + .build()); try { commandLine = commLineParser.parse(options, args); diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index a90a29d9c..d16f7dc04 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -67,8 +67,11 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU final String projectID = commandLineOptions.getOptionValue("p"); final String extractionDir = projectID + "/image-requests/" + extractionName; + final boolean isIdentifiableExtraction = commandLineOptions.hasOption("i"); + _logger.debug("projectID: " + projectID); _logger.debug("extractionDirectory: " + extractionDir); + _logger.debug("isIdentifiableExtraction: " + isIdentifiableExtraction); Path fullExtractionDirectory = Paths.get(extractionRoot.getAbsolutePath().toString(), extractionDir); diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java index f3aa28301..7c993b932 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java @@ -29,6 +29,7 @@ public class ExtractMessagesCsvHandler implements CsvHandler { private String _projectID; private String _extractionDir; private String _extractionModality; + private boolean _isIdentifiableExtraction; private ExtractionKey _extractionKey; private static final Pattern _chiPattern = Pattern.compile("^\\d{10}$"); private static final Pattern _eupiPattern = Pattern.compile("^([A-Z]|[0-9]){32}$"); @@ -51,7 +52,7 @@ public class ExtractMessagesCsvHandler implements CsvHandler { * ExtractRequestInfo messages */ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String extractionDir, - String extractionModality, IProducerModel extractRequestMessageProducerModel, + String extractionModality, boolean isIdentifiableExtraction, IProducerModel extractRequestMessageProducerModel, IProducerModel extractRequestInfoMessageProducerModel) { _extractionJobID = extractionJobID; @@ -60,6 +61,7 @@ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String _extractRequestMessageProducerModel = extractRequestMessageProducerModel; _extractRequestInfoMessageProducerModel = extractRequestInfoMessageProducerModel; _extractionModality = extractionModality; + _isIdentifiableExtraction = isIdentifiableExtraction; // TODO(rkm 2020-01-30) Properly handle parsing of the supported modalities if (_extractionModality != null && (!_extractionModality.equals("CT") && !_extractionModality.equals("MR"))) { @@ -147,6 +149,7 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erm.ExtractionDirectory = _extractionDir; erm.JobSubmittedAt = now; erm.KeyTag = _extractionKey.toString(); + erm.IsIdentifiableExtraction = _isIdentifiableExtraction; if (_extractionKey == ExtractionKey.StudyInstanceUID) erm.ExtractionModality = _extractionModality; @@ -158,6 +161,7 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erim.JobSubmittedAt = now; erim.KeyValueCount = _identifierSet.size(); erim.KeyTag = _extractionKey.toString(); + erim.IsIdentifiableExtraction = _isIdentifiableExtraction; if (_extractionKey == ExtractionKey.StudyInstanceUID) erim.ExtractionModality = _extractionModality; @@ -166,6 +170,7 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I sb.append(" ProjectNumber: " + _projectID + System.lineSeparator()); sb.append(" ExtractionDirectory: " + _extractionDir + System.lineSeparator()); sb.append(" ExtractionKey: " + _extractionKey + System.lineSeparator()); + sb.append(" IsIdentifiableExtraction: " + _isIdentifiableExtraction + System.lineSeparator()); if (_extractionKey == ExtractionKey.StudyInstanceUID) sb.append(" ExtractionModality: " + _extractionModality + System.lineSeparator()); sb.append(" KeyValueCount: " + _identifierSet.size() + System.lineSeparator()); diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java index 61b9e12b4..e73e97938 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java @@ -32,6 +32,12 @@ public abstract class ExtractMessage implements IMessage { @FieldRequired public String JobSubmittedAt; + /** + * True if this is an identifiable extraction (i.e. files should not be anonymised) + */ + @FieldRequired + public boolean IsIdentifiableExtraction; + protected ExtractMessage() { } } From 2df8b894d65d1df235e00306893c0552989e6c6e Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 22:57:06 +0100 Subject: [PATCH 052/138] CTP should throw on messages with IsIdentifiableExtraction set --- .../smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java index 0ae68590b..19a062f4f 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java @@ -72,6 +72,13 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope return; } + if (extractFileMessage.IsIdentifiableExtraction) { + // We should only receive these messages if the queue configuration is wrong, so ok just to crash-out + String msg = "Received a message with IsIdentifiableExtraction set"; + _logger.error(msg); + throw new RuntimeException(msg); + } + ExtractFileStatusMessage statusMessage = new ExtractFileStatusMessage(extractFileMessage); // Got the message, now apply the anonymisation From 12b1645789479e6754436242047258dfd9d10462 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 23:01:10 +0100 Subject: [PATCH 053/138] FileCopier should throw when IsIdentifiableExtraction not set --- .../Messaging/FileCopyQueueConsumer.cs | 3 +++ .../Messaging/FileCopyQueueConsumerTest.cs | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs index a8a2493e6..e365805b1 100644 --- a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs +++ b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs @@ -22,6 +22,9 @@ protected override void ProcessMessageImpl( [NotNull] ExtractFileMessage message, ulong tag) { + if (!message.IsIdentifiableExtraction) + throw new ArgumentException("Received a message with IsIdentifiableExtraction not set"); + try { _fileCopier.ProcessMessage(message, header); diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs index 9aa4f8e99..a15e77c6a 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -52,6 +52,12 @@ public void Test_FileCopyQueueConsumer_UnknownException_CallsFatalCallback() Assert.Inconclusive(); } + [Test] + public void Test_FileCopyQueueConsumer_AnonExtraction_ThrowsException() + { + Assert.Inconclusive(); + } + #endregion } } From 536598dde48c0996d0452468d9f8d6c9777d499e Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 23:04:18 +0100 Subject: [PATCH 054/138] Fix missing arg --- .../java/org/smi/extractorcl/execution/ExtractorClHost.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index d16f7dc04..4390b38f9 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -88,7 +88,7 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionModality = commandLineOptions.getOptionValue("modality", null); - _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, + _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, isIdentifiableExtraction rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestProducerOptions), rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestInfoProducerOptions)); } From d2107f4958e7eec9519f54e6a563bfd9def86d24 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 23:04:18 +0100 Subject: [PATCH 055/138] Fix missing arg --- .../extractorcl/execution/ExtractorClHost.java | 2 +- .../test/execution/ExtractImagesTest.java | 4 ++++ .../fileUtils/ExtractImagesCsvHandlerTest.java | 16 ++++++++-------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index d16f7dc04..90f7d8c79 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -88,7 +88,7 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionModality = commandLineOptions.getOptionValue("modality", null); - _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, + _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, isIdentifiableExtraction, rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestProducerOptions), rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestInfoProducerOptions)); } diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java index 17ab5a178..6b640ac8b 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java @@ -90,6 +90,7 @@ public void testExtractImagesFromSingleFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -152,6 +153,7 @@ public void testExtractImagesFromMultipleFiles() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -214,6 +216,7 @@ public void testMissingFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, null, null); @@ -242,6 +245,7 @@ public void testEmptyFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, null, null); diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java index 8c64c76bc..7b2f14501 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java @@ -27,7 +27,7 @@ public void testProcessingSingleFile() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -83,7 +83,7 @@ public void testProcessingSingleFileWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -138,7 +138,7 @@ public void testProcessingMultipleFilesWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -206,7 +206,7 @@ public void testIdentifierSplit() throws LineProcessingException { UUID extractionUid = UUID.randomUUID(); ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(extractionUid, "MyProjectID", - "MyProjectFolder", null, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); + "MyProjectFolder", null,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -251,14 +251,14 @@ public void testModalityRequirement() throws LineProcessingException { boolean thrown = false; try { - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa",false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); } catch (IllegalArgumentException e) { thrown = true; } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); thrown = false; @@ -269,7 +269,7 @@ public void testModalityRequirement() throws LineProcessingException { } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); @@ -283,7 +283,7 @@ public void testModalityRequirement() throws LineProcessingException { // Happy path - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); handler.processLine(1, new String[] { "s1" }); From 2ba8ed1a48a734f74f946d6eed26085d24c5b6d9 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 23:42:20 +0000 Subject: [PATCH 056/138] fix Java tests --- data/microserviceConfigs/default.yaml | 2 +- .../test/execution/CTPAnonymiserHostTest.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index dad2888d8..ab03a3f92 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -167,7 +167,7 @@ CTPAnonymiserOptions: QoSPrefetchCount: 1 AutoAck: false ExtractFileStatusProducerOptions: - ExchangeName: 'TEST.FileStatusExchange' + ExchangeName: 'TEST.ExtractedFileStatusExchange' MaxConfirmAttempts: 1 FileCopierOptions: diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index 9e78aaa0e..340428e52 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -103,7 +103,7 @@ protected void setUp() throws Exception { _channel.exchangeDeclare(_inputExchName, "direct", true); _channel.queueDeclare(_consumerQueueName, true, false, false, null); - _channel.queueBind(_consumerQueueName, _inputExchName, ""); + _channel.queueBind(_consumerQueueName, _inputExchName, "anon"); System.out.println(String.format("Bound %s -> %s", _inputExchName, _consumerQueueName)); // Setup the output exch. / queue pair @@ -111,8 +111,8 @@ protected void setUp() throws Exception { _channel.exchangeDeclare(_producerExchangeName, "direct", true); _channel.queueDeclare(_outputQueueName, true, false, false, null); _channel.queueBind(_outputQueueName, _producerExchangeName, ""); - _channel.queueBind(_outputQueueName, _producerExchangeName, "success"); - _channel.queueBind(_outputQueueName, _producerExchangeName, "failure"); + _channel.queueBind(_outputQueueName, _producerExchangeName, "verify"); + _channel.queueBind(_outputQueueName, _producerExchangeName, "noverify"); System.out.println(String.format("Bound %s -> %s", _producerExchangeName, _outputQueueName)); _channel.queuePurge(_consumerQueueName); @@ -164,7 +164,7 @@ public void testBasicAnonymise_Success() throws InterruptedException { TimeUnit.MILLISECONDS.sleep(1000); _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + _extractFileMessageProducer.SendMessage(exMessage, "anon", null); _logger.info("Waiting..."); @@ -214,7 +214,7 @@ public void testBasicAnonymise_Failure() throws InterruptedException { exMessage.ProjectNumber = "123-456"; _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + _extractFileMessageProducer.SendMessage(exMessage, "anon", null); _logger.info("Waiting..."); From 63a8af3f153b2d045e6a0627940f91acba227324 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 26 Aug 2020 23:55:19 +0000 Subject: [PATCH 057/138] Fix missing exchange This is suddenly needed? --- .../ServiceTests/IsIdentifiableHostTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs index 241c3cb28..5da01b81f 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs @@ -53,6 +53,8 @@ public void TestClassifierName_ValidClassifier() using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions)) { + tester.CreateExchange(options.IsIdentifiableOptions.IsIdentifiableProducerOptions.ExchangeName, null); + options.IsIdentifiableOptions.ClassifierType = typeof(RejectAllClassifier).FullName; options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.TestDirectory; From adc8756da865348498fcc96f128796efe5a466e2 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 12:34:08 +0100 Subject: [PATCH 058/138] Allow routing keys to be injected to MicroserviceTesters --- .../Smi.Common.Tests/MicroserviceTester.cs | 14 +++++++------- .../Smi.Common.Tests/RequiresRelationalDb.cs | 5 +++-- .../MicroservicesIntegrationTest.cs | 17 +++++++++-------- .../DicomTagReaderTestHelper.cs | 4 ++-- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/common/Smi.Common.Tests/MicroserviceTester.cs b/tests/common/Smi.Common.Tests/MicroserviceTester.cs index 924f0ba65..d681a426d 100644 --- a/tests/common/Smi.Common.Tests/MicroserviceTester.cs +++ b/tests/common/Smi.Common.Tests/MicroserviceTester.cs @@ -1,6 +1,5 @@  using RabbitMQ.Client; -using Smi.Common; using Smi.Common.Execution; using Smi.Common.Messages; using Smi.Common.Messaging; @@ -8,6 +7,7 @@ using System; using System.Collections.Generic; + namespace Smi.Common.Tests { public class MicroserviceTester : IDisposable @@ -137,12 +137,12 @@ public void SendMessage(ConsumerOptions toConsumer, IMessageHeader header, IMess /// false to create an entirely new Exchange=>Queue (including deleting any existing queue/exchange). False to simply declare the /// queue and bind it to the exchange which is assumed to already exist (this allows you to set up exchange=>multiple queues). If you are setting up multiple queues /// from a single exchange the first call should be isSecondaryBinding = false and all further calls after that for the same exchange should be isSecondaryBinding=true - public void CreateExchange(string exchangeName, ConsumerOptions consumerIfAny, bool isSecondaryBinding = false) + public void CreateExchange(string exchangeName, string queueName = null, bool isSecondaryBinding = false, string routingKey = "") { if (!exchangeName.Contains("TEST.")) exchangeName = exchangeName.Insert(0, "TEST."); - string queueName = consumerIfAny != null ? consumerIfAny.QueueName : exchangeName; + string queueNameToUse = queueName ?? exchangeName.Replace("Exchange", "Queue"); using (var con = _factory.CreateConnection()) using (var model = con.CreateModel()) @@ -153,16 +153,16 @@ public void CreateExchange(string exchangeName, ConsumerOptions consumerIfAny, b if (!isSecondaryBinding) model.ExchangeDelete(exchangeName); - model.QueueDelete(queueName); + model.QueueDelete(queueNameToUse); //Create a binding between the exchange and the queue if (!isSecondaryBinding) model.ExchangeDeclare(exchangeName, ExchangeType.Direct, true);//durable seems to be needed because RabbitMQAdapter wants it? - model.QueueDeclare(queueName, true, false, false); //shared with other users - model.QueueBind(queueName, exchangeName, ""); + model.QueueDeclare(queueNameToUse, true, false, false); //shared with other users + model.QueueBind(queueNameToUse, exchangeName, routingKey); - Console.WriteLine("Created Exchange " + exchangeName + "=>" + queueName); + Console.WriteLine("Created Exchange " + exchangeName + "=>" + queueNameToUse); } } diff --git a/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs b/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs index c8744f7e9..977c35354 100644 --- a/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs +++ b/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs @@ -39,10 +39,11 @@ public void ApplyToContext(TestExecutionContext context) if (server.Exists()) return; + string msg = $"Could not connect to {_type} at '{server.Name}' with the provided connection options"; if (!FailIfUnavailable) - Assert.Ignore(_type + " is not running at '" + server.Name + "'"); + Assert.Ignore(msg); else - Assert.Fail(_type + " is not running at '" + server.Name + "'"); + Assert.Fail(msg); } public static ConStrs GetRelationalDatabaseConnectionStrings() diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs index 2bb461f77..a62980c8c 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs @@ -461,14 +461,15 @@ private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action Date: Thu, 27 Aug 2020 12:03:55 +0000 Subject: [PATCH 059/138] fix extraction output directory --- CHANGELOG.md | 1 + .../java/org/smi/extractorcl/execution/ExtractorClHost.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46a715d09..138bc8102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add SecurityCodeScan tool to build chain for .Net code - Extraction report: Group PixelData separately and sort by length +- Fix the extraction output directory to be `/extractions/` ## [1.11.1] - 2020-08-12 diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index a90a29d9c..ccd0fd551 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -65,7 +65,7 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionName = path.getFileName().toString().replaceFirst("[.][^.]+$", ""); final String projectID = commandLineOptions.getOptionValue("p"); - final String extractionDir = projectID + "/image-requests/" + extractionName; + final String extractionDir = projectID + "/extractions/" + extractionName; _logger.debug("projectID: " + projectID); _logger.debug("extractionDirectory: " + extractionDir); From 442b017f1269eeea7d4a7500feff694a3bbc13eb Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 13:49:26 +0100 Subject: [PATCH 060/138] Add IsIdentifiableExtraction to MongoDB extraction metadata --- .../Extraction/ExtractFileStatusMessage.cs | 4 ++-- .../ExtractJobStorage/ExtractJobStore.cs | 2 -- .../MongoDB/ObjectModel/MongoExtractJobDoc.cs | 13 +++++++++-- .../MongoDB/MongoExtractJobStoreTest.cs | 12 ++++++---- .../MongoCompletedExtractJobDocTest.cs | 1 + .../ObjectModel/MongoExtractJobDocTest.cs | 23 ++++++++++++++++--- 6 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs index da665c302..07cc780a5 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs @@ -5,13 +5,14 @@ namespace Smi.Common.Messages.Extraction { /// - /// Status message received from the anonymisation service + /// Status message sent by services which extract files (CTP, FileCopier) /// public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, IEquatable { /// /// Original file path /// + [JsonProperty(Required = Required.Always)] public string DicomFilePath { get; set; } /// @@ -20,7 +21,6 @@ public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, I [JsonProperty(Required = Required.Always)] public ExtractFileStatus Status { get; set; } - // TODO Consider renaming /// /// Anonymised file name. Only required if a file has been anonymised /// diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index c6793d122..ac498f88e 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -43,13 +43,11 @@ public void PersistMessageToStore( [NotNull] ExtractFileStatusMessage message, [NotNull] IMessageHeader header) { - // TODO if (message.Status == ExtractFileStatus.Unknown) throw new ApplicationException("ExtractFileStatus was unknown"); if (message.Status == ExtractFileStatus.Anonymised) throw new ApplicationException("Received an anonymisation successful message from the failure queue"); - // TODO PersistMessageToStoreImpl(message, header); } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs index 7abbaa11d..c0aa79a09 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs @@ -45,6 +45,9 @@ public class MongoExtractJobDoc [CanBeNull] public string ExtractionModality { get; set; } + [BsonElement("isIdentifiableExtraction")] + public bool IsIdentifiableExtraction { get; set; } + [BsonElement("failedJobInfo")] [CanBeNull] public MongoFailedJobInfoDoc FailedJobInfoDoc { get; set; } @@ -60,6 +63,7 @@ public MongoExtractJobDoc( [NotNull] string keyTag, uint keyCount, [CanBeNull] string extractionModality, + bool isIdentifiableExtraction, [CanBeNull] MongoFailedJobInfoDoc failedJobInfoDoc) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentException(nameof(extractionJobIdentifier)); @@ -72,9 +76,11 @@ public MongoExtractJobDoc( KeyCount = (keyCount > 0) ? keyCount : throw new ArgumentNullException(nameof(keyCount)); if (extractionModality != null) ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); + IsIdentifiableExtraction = isIdentifiableExtraction; FailedJobInfoDoc = failedJobInfoDoc; } + /// /// Copy constructor /// @@ -107,6 +113,7 @@ public static MongoExtractJobDoc FromMessage( message.KeyTag, (uint)message.KeyValueCount, message.ExtractionModality, + message.IsIdentifiableExtraction, null ); } @@ -124,6 +131,7 @@ protected bool Equals(MongoExtractJobDoc other) KeyTag == other.KeyTag && KeyCount == other.KeyCount && ExtractionModality == other.ExtractionModality && + IsIdentifiableExtraction == other.IsIdentifiableExtraction && Equals(FailedJobInfoDoc, other.FailedJobInfoDoc); } @@ -146,12 +154,13 @@ public override int GetHashCode() int hashCode = ExtractionJobIdentifier.GetHashCode(); hashCode = (hashCode * 397) ^ Header.GetHashCode(); hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); - hashCode = (hashCode * 397) ^ (int)JobStatus; + hashCode = (hashCode * 397) ^ (int) JobStatus; hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode(); hashCode = (hashCode * 397) ^ KeyTag.GetHashCode(); - hashCode = (hashCode * 397) ^ (int)KeyCount; + hashCode = (hashCode * 397) ^ (int) KeyCount; hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); hashCode = (hashCode * 397) ^ (FailedJobInfoDoc != null ? FailedJobInfoDoc.GetHashCode() : 0); return hashCode; } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs index 76de61864..2661775e1 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs @@ -1,7 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; @@ -16,6 +12,10 @@ using Smi.Common.MongoDb.Tests; using Smi.Common.MongoDB.Tests; using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB { @@ -253,6 +253,7 @@ public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() "StudyInstanceUID", 1, "CT", + isIdentifiableExtraction: false, null); Assert.AreEqual(expected, extractJob); @@ -439,6 +440,7 @@ public void TestGetReadJobsImpl() "SeriesInstanceUID", 1, "MR", + isIdentifiableExtraction: false, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -514,6 +516,7 @@ public void TestCompleteJobImpl() "SeriesInstanceUID", 1, "MR", + isIdentifiableExtraction: false, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -620,6 +623,7 @@ public void TestMarkJobFailedImpl() "1.2.3.4", 123, "MR", + isIdentifiableExtraction: false, null); var client = new TestMongoClient(); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs index 27cbf6f42..371635459 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs @@ -24,6 +24,7 @@ public class MongoCompletedExtractJobDocTest "test", 1, null, + isIdentifiableExtraction: false, null); #region Fixture Methods diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs index 98ba37c8d..69de742f3 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs @@ -1,12 +1,14 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -76,6 +78,7 @@ public void TestMongoExtractJobDoc_FromMessage() "KeyTag", 123, "MR", + isIdentifiableExtraction: false, null); Assert.AreEqual(expected, doc); @@ -97,6 +100,7 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", + isIdentifiableExtraction: false, failedInfoDoc); var doc2 = new MongoExtractJobDoc( guid, @@ -108,6 +112,7 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", + isIdentifiableExtraction: false, failedInfoDoc); Assert.AreEqual(doc1, doc2); @@ -128,6 +133,7 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", + isIdentifiableExtraction: false, null); var doc2 = new MongoExtractJobDoc( guid, @@ -139,11 +145,22 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", + isIdentifiableExtraction: false, null); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); } + [Test] + public void TestMongoExtractJobDoc_IsIdentifiableExtraction_MissingValueOk() + { + // TODO(rkm 2020-08-27) This works by chance since the missing boolean value defaults to false anyway. Need to think of a better way of handling this kind of backwards compatibility + var jsonDoc = "{ \"_id\" : \"0fbd4893-c116-4f16-88c8-f7084531d87c\", \"header\" : { \"extractionJobIdentifier\" : \"0fbd4893-c116-4f16-88c8-f7084531d87c\", \"messageGuid\" : \"23475d89-6e2c-431c-bc5d-7b7c25ffb6a0\", \"producerExecutableName\" : \"testhost\", \"producerProcessID\" : 14372, \"originalPublishTimestamp\" : { \"$date\" : 1598528178000 }, \"parents\" : \"30603fb0-3bec-43de-8ba8-db55a029c664\", \"receivedAt\" : { \"$date\" : 1598531778957 } }, \"projectNumber\" : \"1234\", \"jobStatus\" : \"WaitingForCollectionInfo\", \"extractionDirectory\" : \"test/directory\", \"jobSubmittedAt\" : { \"$date\" : 1598531778957 }, \"keyTag\" : \"KeyTag\", \"keyCount\" : 123, \"extractionModality\" : \"MR\",\"failedJobInfo\" : null }"; + BsonDocument bsonDoc = BsonDocument.Parse(jsonDoc); + var mongoExtractJobDoc = BsonSerializer.Deserialize(bsonDoc); + Assert.False(mongoExtractJobDoc.IsIdentifiableExtraction); + } + [Test] public void TestMongoFailedJobInfoDoc_SettersAvailable() { From 30b5153ef5ff87af7dc6acd6fc78602f2ae80bf0 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 14:09:01 +0100 Subject: [PATCH 061/138] Rename message and property to match new format --- ...ssage.cs => ExtractedFileStatusMessage.cs} | 25 ++++++++++--------- .../Extraction/IsIdentifiableMessage.cs | 12 ++++----- .../ExtractJobStorage/ExtractJobStore.cs | 6 ++--- .../ExtractJobStorage/IExtractJobStore.cs | 6 ++--- .../MongoDB/MongoExtractJobStore.cs | 8 +++--- .../Messaging/AnonFailedMessageConsumer.cs | 8 +++--- .../Execution/ExtractionFileCopier.cs | 8 +++--- .../Messaging/FileCopyQueueConsumer.cs | 2 +- .../Service/IsIdentifiableQueueConsumer.cs | 6 ++--- ...e.java => ExtractedFileStatusMessage.java} | 8 +++--- .../messaging/CTPAnonymiserConsumer.java | 6 ++--- .../test/execution/CTPAnonymiserHostTest.java | 14 +++++------ .../Execution/CohortPackagerHostTest.cs | 8 +++--- .../ExtractJobStorage/ExtractJobStoreTest.cs | 12 ++++----- .../MongoDB/MongoExtractJobStoreTest.cs | 6 ++--- .../Execution/FileCopierTest.cs | 24 +++++++++--------- .../ServiceTests/IsIdentifiableHostTests.cs | 8 +++--- 17 files changed, 84 insertions(+), 83 deletions(-) rename src/common/Smi.Common/Messages/Extraction/{ExtractFileStatusMessage.cs => ExtractedFileStatusMessage.cs} (70%) rename src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/{ExtractFileStatusMessage.java => ExtractedFileStatusMessage.java} (87%) diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs similarity index 70% rename from src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs rename to src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs index 07cc780a5..feba42d4b 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs @@ -7,7 +7,7 @@ namespace Smi.Common.Messages.Extraction /// /// Status message sent by services which extract files (CTP, FileCopier) /// - public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, IEquatable + public class ExtractedFileStatusMessage : ExtractMessage, IFileReferenceMessage, IEquatable { /// /// Original file path @@ -22,10 +22,10 @@ public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, I public ExtractFileStatus Status { get; set; } /// - /// Anonymised file name. Only required if a file has been anonymised + /// Output file path, relative to the extraction directory. Only required if an output file has been produced /// [JsonProperty(Required = Required.DisallowNull)] - public string AnonymisedFileName { get; set; } + public string OutputFilePath { get; set; } /// /// Message required if Status is not 0 @@ -34,30 +34,31 @@ public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, I public string StatusMessage { get; set; } + // TODO [JsonConstructor] - public ExtractFileStatusMessage() { } + public ExtractedFileStatusMessage() { } - public ExtractFileStatusMessage(IExtractMessage request) + public ExtractedFileStatusMessage(IExtractMessage request) : base(request) { } public override string ToString() => $"{base.ToString()}," + $"DicomFilePath={DicomFilePath}," + $"ExtractFileStatus={Status}," + - $"AnonymisedFileName={AnonymisedFileName}," + + $"OutputFilePath={OutputFilePath}," + $"StatusMessage={StatusMessage}," + ""; #region Equality Members - public bool Equals(ExtractFileStatusMessage other) + public bool Equals(ExtractedFileStatusMessage other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; return base.Equals(other) && Status == other.Status && - string.Equals(AnonymisedFileName, other.AnonymisedFileName) && + string.Equals(OutputFilePath, other.OutputFilePath) && string.Equals(StatusMessage, other.StatusMessage); } @@ -66,7 +67,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((ExtractFileStatusMessage)obj); + return Equals((ExtractedFileStatusMessage)obj); } public override int GetHashCode() @@ -76,18 +77,18 @@ public override int GetHashCode() int hashCode = base.GetHashCode(); hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (int)Status; - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFilePath != null ? OutputFilePath.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (StatusMessage != null ? StatusMessage.GetHashCode() : 0); return hashCode; } } - public static bool operator ==(ExtractFileStatusMessage left, ExtractFileStatusMessage right) + public static bool operator ==(ExtractedFileStatusMessage left, ExtractedFileStatusMessage right) { return Equals(left, right); } - public static bool operator !=(ExtractFileStatusMessage left, ExtractFileStatusMessage right) + public static bool operator !=(ExtractedFileStatusMessage left, ExtractedFileStatusMessage right) { return !Equals(left, right); } diff --git a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs b/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs index f5515bc71..e08f8f8ea 100644 --- a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs @@ -19,10 +19,10 @@ public class IsIdentifiableMessage : ExtractMessage, IFileReferenceMessage, IEqu public string DicomFilePath { get; set; } /// - /// Anonymised file name. Only required if a file has been anonymised + /// Output file path, relative to the extraction directory. Only required if an output file has been produced /// [JsonProperty(Required = Required.Always)] - public string AnonymisedFileName { get; set; } + public string OutputFilePath { get; set; } [JsonConstructor] public IsIdentifiableMessage() { } @@ -40,12 +40,12 @@ public IsIdentifiableMessage(Guid extractionJobIdentifier, string projectNumber, /// Creates a new instance copying all values from the given origin message /// /// - public IsIdentifiableMessage(ExtractFileStatusMessage request) + public IsIdentifiableMessage(ExtractedFileStatusMessage request) : this(request.ExtractionJobIdentifier, request.ProjectNumber, request.ExtractionDirectory, request.JobSubmittedAt) { DicomFilePath = request.DicomFilePath; - AnonymisedFileName = request.AnonymisedFileName; + OutputFilePath = request.OutputFilePath; } #region Equality Members @@ -54,7 +54,7 @@ public bool Equals(IsIdentifiableMessage other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return base.Equals(other) && IsIdentifiable == other.IsIdentifiable && DicomFilePath == other.DicomFilePath && AnonymisedFileName == other.AnonymisedFileName; + return base.Equals(other) && IsIdentifiable == other.IsIdentifiable && DicomFilePath == other.DicomFilePath && OutputFilePath == other.OutputFilePath; } public override bool Equals(object obj) @@ -72,7 +72,7 @@ public override int GetHashCode() int hashCode = base.GetHashCode(); hashCode = (hashCode * 397) ^ IsIdentifiable.GetHashCode(); hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFilePath != null ? OutputFilePath.GetHashCode() : 0); return hashCode; } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index ac498f88e..51038ea1e 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -40,7 +40,7 @@ public void PersistMessageToStore(ExtractFileCollectionInfoMessage message, IMes } public void PersistMessageToStore( - [NotNull] ExtractFileStatusMessage message, + [NotNull] ExtractedFileStatusMessage message, [NotNull] IMessageHeader header) { if (message.Status == ExtractFileStatus.Unknown) @@ -55,7 +55,7 @@ public void PersistMessageToStore( [NotNull] IsIdentifiableMessage message, [NotNull] IMessageHeader header) { - if (string.IsNullOrWhiteSpace(message.AnonymisedFileName)) + if (string.IsNullOrWhiteSpace(message.OutputFilePath)) throw new ApplicationException("Received a verification message without the AnonymisedFileName set"); if (string.IsNullOrWhiteSpace(message.Report)) throw new ApplicationException("Null or empty report data"); @@ -127,7 +127,7 @@ public IEnumerable> GetCompletedJobVerificationFailures(Gu protected abstract void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header); protected abstract void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header); - protected abstract void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header); + protected abstract void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header); protected abstract void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header); protected abstract List GetReadyJobsImpl(Guid specificJobId = new Guid()); protected abstract void CompleteJobImpl(Guid jobId); diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs index a6d8de53a..970558130 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs @@ -26,14 +26,14 @@ public interface IExtractJobStore void PersistMessageToStore(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header); /// - /// Serializes a and it's and stores it + /// Serializes a and it's and stores it /// /// /// - void PersistMessageToStore(ExtractFileStatusMessage fileStatusMessage, IMessageHeader header); + void PersistMessageToStore(ExtractedFileStatusMessage fileStatusMessage, IMessageHeader header); /// - /// Serializes a and it's and stores it + /// Serializes a and it's and stores it /// /// /// diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs index 051316527..b9160f322 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs @@ -70,14 +70,14 @@ protected override void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessa .InsertOne(expectedFilesForKey); } - protected override void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header) + protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header) { if (InCompletedJobCollection(message.ExtractionJobIdentifier)) - throw new ApplicationException("Received an ExtractFileStatusMessage for a job that is already completed"); + throw new ApplicationException("Received an ExtractedFileStatusMessage for a job that is already completed"); var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), - message.AnonymisedFileName, + message.OutputFilePath, wasAnonymised: false, isIdentifiable: true, statusMessage: message.StatusMessage); @@ -94,7 +94,7 @@ protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), - message.AnonymisedFileName, + message.OutputFilePath, wasAnonymised: true, isIdentifiable: message.IsIdentifiable, statusMessage: message.Report); diff --git a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs index 1431c419a..dfa582d64 100644 --- a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs +++ b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs @@ -10,9 +10,9 @@ namespace Microservices.CohortPackager.Messaging { // TODO Naming /// - /// Consumer for (s) + /// Consumer for (s) /// - public class AnonFailedMessageConsumer : Consumer + public class AnonFailedMessageConsumer : Consumer { private readonly IExtractJobStore _store; @@ -22,7 +22,7 @@ public AnonFailedMessageConsumer(IExtractJobStore store) _store = store; } - protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileStatusMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileStatusMessage message, ulong tag) { try { @@ -31,7 +31,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileSta catch (ApplicationException e) { // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage - ErrorAndNack(header, tag, "Error while processing ExtractFileStatusMessage", e); + ErrorAndNack(header, tag, "Error while processing ExtractedFileStatusMessage", e); return; } diff --git a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs index 0342fba75..e8f73bb84 100644 --- a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -42,11 +42,11 @@ public void ProcessMessage( { string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath); - ExtractFileStatusMessage statusMessage; + ExtractedFileStatusMessage statusMessage; if (!_fileSystem.File.Exists(fullSrc)) { - statusMessage = new ExtractFileStatusMessage(message) + statusMessage = new ExtractedFileStatusMessage(message) { DicomFilePath = message.DicomFilePath, Status = ExtractFileStatus.FileMissing, @@ -71,11 +71,11 @@ public void ProcessMessage( _logger.Debug($"Copying source file to '{message.OutputPath}'"); _fileSystem.File.Copy(fullSrc, fullDest, overwrite: true); - statusMessage = new ExtractFileStatusMessage(message) + statusMessage = new ExtractedFileStatusMessage(message) { DicomFilePath = message.DicomFilePath, Status = ExtractFileStatus.Copied, - AnonymisedFileName = message.OutputPath, + OutputFilePath = message.OutputPath, }; _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); } diff --git a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs index e365805b1..e7677aae3 100644 --- a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs +++ b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs @@ -32,7 +32,7 @@ protected override void ProcessMessageImpl( catch (ApplicationException e) { // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage - ErrorAndNack(header, tag, "Error while processing ExtractFileStatusMessage", e); + ErrorAndNack(header, tag, "Error while processing ExtractedFileStatusMessage", e); return; } diff --git a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs index 5a387d237..44b13fa2b 100644 --- a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs +++ b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs @@ -11,7 +11,7 @@ namespace Microservices.IsIdentifiable.Service { - public class IsIdentifiableQueueConsumer : Consumer, IDisposable + public class IsIdentifiableQueueConsumer : Consumer, IDisposable { private readonly IProducerModel _producer; private readonly string _fileSystemRoot; @@ -26,7 +26,7 @@ public IsIdentifiableQueueConsumer(IProducerModel producer, string fileSystemRoo _classifier = classifier; } - protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileStatusMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileStatusMessage message, ulong tag) { bool isClean = true; object resultObject; @@ -37,7 +37,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileSta if (message.Status != ExtractFileStatus.Anonymised) throw new ApplicationException($"Received a message with anonymised status of {message.Status}"); - var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.AnonymisedFileName) ); + var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputFilePath) ); if(!toProcess.Exists) throw new ApplicationException("IsIdentifiable service cannot find file "+toProcess.FullName); diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java similarity index 87% rename from src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java rename to src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java index 268c46b9f..d9c48c330 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java @@ -8,12 +8,12 @@ /** * Message indicating the path to an anonymised file */ -public class ExtractFileStatusMessage extends ExtractMessage implements IMessage { +public class ExtractedFileStatusMessage extends ExtractMessage implements IMessage { @FieldRequired public String DicomFilePath; - public String AnonymisedFileName; + public String OutputFilePath; @FieldRequired public ExtractFileStatus Status; @@ -46,7 +46,7 @@ public void setProjectNumber(String projectNumber) { ProjectNumber = projectNumber; } - public ExtractFileStatusMessage(ExtractFileMessage request) { + public ExtractedFileStatusMessage(ExtractFileMessage request) { ExtractionJobIdentifier = request.ExtractionJobIdentifier; ExtractionDirectory = request.ExtractionDirectory; @@ -65,7 +65,7 @@ public String toString() { sb.append("DicomFilePath: " + DicomFilePath + "\n"); sb.append("ProjectNumber: " + ProjectNumber + "\n"); sb.append("JobSubmittedAt: " + JobSubmittedAt + "\n"); - sb.append("AnonymisedFileName: " + AnonymisedFileName + "\n"); + sb.append("OutputFilePath: " + OutputFilePath + "\n"); sb.append("Status: " + Status + "\n"); sb.append("StatusMessage: " + StatusMessage + "\n"); diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java index 19a062f4f..936ae78d6 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java @@ -11,7 +11,7 @@ import org.smi.common.messaging.SmiConsumer; import org.smi.ctpanonymiser.execution.SmiCtpProcessor; import org.smi.ctpanonymiser.messages.ExtractFileMessage; -import org.smi.ctpanonymiser.messages.ExtractFileStatusMessage; +import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; import org.smi.ctpanonymiser.util.CtpAnonymisationStatus; import org.smi.common.options.GlobalOptions; import org.smi.ctpanonymiser.util.ExtractFileStatus; @@ -79,7 +79,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope throw new RuntimeException(msg); } - ExtractFileStatusMessage statusMessage = new ExtractFileStatusMessage(extractFileMessage); + ExtractedFileStatusMessage statusMessage = new ExtractedFileStatusMessage(extractFileMessage); // Got the message, now apply the anonymisation @@ -155,7 +155,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope if (status == CtpAnonymisationStatus.Anonymised) { - statusMessage.AnonymisedFileName = extractFileMessage.OutputPath; + statusMessage.OutputFilePath = extractFileMessage.OutputPath; statusMessage.Status = ExtractFileStatus.Anonymised; routingKey = _routingKey_success; diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index 340428e52..d93a228c2 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -16,7 +16,7 @@ import org.smi.ctpanonymiser.Program; import org.smi.ctpanonymiser.execution.CTPAnonymiserHost; import org.smi.ctpanonymiser.messages.ExtractFileMessage; -import org.smi.ctpanonymiser.messages.ExtractFileStatusMessage; +import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; import org.smi.ctpanonymiser.util.ExtractFileStatus; import java.io.File; @@ -41,7 +41,7 @@ public class CTPAnonymiserHostTest extends TestCase { private ProducerOptions _extractFileProducerOptions; private ConsumerOptions _extractFileStatusConsumerOptions; private IProducerModel _extractFileMessageProducer; - private AnyConsumer _anonFileStatusMessageConsumer; + private AnyConsumer _anonFileStatusMessageConsumer; private ConnectionFactory _factory; private Connection _conn; @@ -88,7 +88,7 @@ protected void setUp() throws Exception { _extractFileStatusConsumerOptions.AutoAck = false; _extractFileStatusConsumerOptions.QoSPrefetchCount = 1; - _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractFileStatusMessage.class); + _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractedFileStatusMessage.class); _extractFileProducerOptions = new ProducerOptions(); _extractFileProducerOptions.ExchangeName = _inputExchName; @@ -185,12 +185,12 @@ public void testBasicAnonymise_Success() throws InterruptedException { if (_anonFileStatusMessageConsumer.isMessageValid()) { - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + ExtractedFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); _logger.info("Message received"); _logger.info("\n" + recvd.toString()); - assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.AnonymisedFileName); + assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.OutputFilePath); assertEquals("Project numbers do not match", exMessage.ProjectNumber, recvd.ProjectNumber); assertEquals(ExtractFileStatus.Anonymised, recvd.Status); } else { @@ -235,12 +235,12 @@ public void testBasicAnonymise_Failure() throws InterruptedException { if (_anonFileStatusMessageConsumer.isMessageValid()) { - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + ExtractedFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); _logger.info("Message received"); _logger.info("\n" + recvd.toString()); - assertEquals("FilePaths do not match", null, recvd.AnonymisedFileName); + assertEquals("FilePaths do not match", null, recvd.OutputFilePath); assertEquals(ExtractFileStatus.ErrorWontRetry, recvd.Status); } else { fail("Did not receive message"); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs index 1f982a1ad..170404b1e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs @@ -65,7 +65,7 @@ public void NotifyJobCompleted(ExtractJobInfo jobInfo) } [Test] - public void TestCohortPackagerHost_HappyPath() + public void Test_CohortPackagerHost_HappyPath() { Guid jobId = Guid.NewGuid(); var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage @@ -95,10 +95,10 @@ public void TestCohortPackagerHost_HappyPath() }, KeyValue = "study-1", }; - var testExtractFileStatusMessage = new ExtractFileStatusMessage + var testExtractFileStatusMessage = new ExtractedFileStatusMessage { JobSubmittedAt = DateTime.UtcNow, - AnonymisedFileName = "study-1-anon-1.dcm", + OutputFilePath = "study-1-anon-1.dcm", ProjectNumber = "testProj1", ExtractionJobIdentifier = jobId, ExtractionDirectory = "test", @@ -109,7 +109,7 @@ public void TestCohortPackagerHost_HappyPath() var testIsIdentifiableMessage = new IsIdentifiableMessage { JobSubmittedAt = DateTime.UtcNow, - AnonymisedFileName = "study-1-anon-2.dcm", + OutputFilePath = "study-1-anon-2.dcm", ProjectNumber = "testProj1", ExtractionJobIdentifier = jobId, ExtractionDirectory = "test", diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs index b99d67df8..ab8bfdfeb 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs @@ -34,7 +34,7 @@ private class TestExtractJobStore : ExtractJobStore { protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header) { } protected override void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header) => throw new NotImplementedException(); - protected override void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header) { } + protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header) { } protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header) { } protected override List GetReadyJobsImpl(Guid specificJobId = new Guid()) => throw new NotImplementedException(); protected override void CompleteJobImpl(Guid jobId) { } @@ -77,7 +77,7 @@ public void TestPersistMessageToStore_ExtractionRequestInfoMessage() public void TestPersistMessageToStore_ExtractFileStatusMessage() { var testExtractJobStore = new TestExtractJobStore(); - var message = new ExtractFileStatusMessage(); + var message = new ExtractedFileStatusMessage(); var header = new MessageHeader(); message.Status = ExtractFileStatus.Unknown; @@ -99,18 +99,18 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() // Must have AnonymisedFileName var message = new IsIdentifiableMessage(); - message.AnonymisedFileName = ""; + message.OutputFilePath = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report shouldn't be an empty string or null message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message.OutputFilePath = "anon.dcm"; message.Report = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report needs to contain content if marked as IsIdentifiable message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = true; message.Report = "[]"; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); @@ -120,7 +120,7 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() // Report can be empty if not marked as IsIdentifiable message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = false; message.Report = "[]"; testExtractJobStore.PersistMessageToStore(message, header); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs index 2661775e1..ea9a27e93 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs @@ -360,9 +360,9 @@ public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); - var testExtractFileStatusMessage = new ExtractFileStatusMessage + var testExtractFileStatusMessage = new ExtractedFileStatusMessage { - AnonymisedFileName = "anon.dcm", + OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), Status = ExtractFileStatus.ErrorWontRetry, ProjectNumber = "1234", @@ -399,7 +399,7 @@ public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() Guid jobId = Guid.NewGuid(); var testIsIdentifiableMessage = new IsIdentifiableMessage { - AnonymisedFileName = "anon.dcm", + OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), ProjectNumber = "1234", ExtractionJobIdentifier = jobId, diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs index ebe4fdef8..ebf4e3fb7 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -73,13 +73,13 @@ public void TearDown() { } public void Test_FileCopier_HappyPath() { var mockProducerModel = new Mock(MockBehavior.Strict); - ExtractFileStatusMessage sentStatusMessage = null; + ExtractedFileStatusMessage sentStatusMessage = null; string sentRoutingKey = null; mockProducerModel .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) .Callback((IMessage message, IMessageHeader header, string routingKey) => { - sentStatusMessage = (ExtractFileStatusMessage)message; + sentStatusMessage = (ExtractedFileStatusMessage)message; sentRoutingKey = routingKey; }) .Returns(() => null); @@ -89,11 +89,11 @@ public void Test_FileCopier_HappyPath() var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); - var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, Status = ExtractFileStatus.Copied, - AnonymisedFileName = _requestMessage.OutputPath, + OutputFilePath = _requestMessage.OutputPath, }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); @@ -107,13 +107,13 @@ public void Test_FileCopier_HappyPath() public void Test_FileCopier_MissingFile_SendsMessage() { var mockProducerModel = new Mock(MockBehavior.Strict); - ExtractFileStatusMessage sentStatusMessage = null; + ExtractedFileStatusMessage sentStatusMessage = null; string sentRoutingKey = null; mockProducerModel .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) .Callback((IMessage message, IMessageHeader header, string routingKey) => { - sentStatusMessage = (ExtractFileStatusMessage)message; + sentStatusMessage = (ExtractedFileStatusMessage)message; sentRoutingKey = routingKey; }) .Returns(() => null); @@ -124,11 +124,11 @@ public void Test_FileCopier_MissingFile_SendsMessage() var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); - var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, Status = ExtractFileStatus.FileMissing, - AnonymisedFileName = null, + OutputFilePath = null, StatusMessage = $"Could not find '{_mockFileSystem.Path.Combine(FileSystemRoot, "missing.dcm")}'" }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); @@ -139,13 +139,13 @@ public void Test_FileCopier_MissingFile_SendsMessage() public void Test_FileCopier_ExistingOutputFile_IsOverwritten() { var mockProducerModel = new Mock(MockBehavior.Strict); - ExtractFileStatusMessage sentStatusMessage = null; + ExtractedFileStatusMessage sentStatusMessage = null; string sentRoutingKey = null; mockProducerModel .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) .Callback((IMessage message, IMessageHeader header, string routingKey) => { - sentStatusMessage = (ExtractFileStatusMessage)message; + sentStatusMessage = (ExtractedFileStatusMessage)message; sentRoutingKey = routingKey; }) .Returns(() => null); @@ -158,11 +158,11 @@ public void Test_FileCopier_ExistingOutputFile_IsOverwritten() var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); copier.ProcessMessage(_requestMessage, requestHeader); - var expectedStatusMessage = new ExtractFileStatusMessage(_requestMessage) + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, Status = ExtractFileStatus.Copied, - AnonymisedFileName = _requestMessage.OutputPath, + OutputFilePath = _requestMessage.OutputPath, StatusMessage = null, }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs index 5da01b81f..034cbd631 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs @@ -62,10 +62,10 @@ public void TestClassifierName_ValidClassifier() Assert.IsNotNull(host); host.Start(); - tester.SendMessage(options.IsIdentifiableOptions, new ExtractFileStatusMessage() + tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage() { DicomFilePath = "yay.dcm", - AnonymisedFileName = testDcm.FullName, + OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", @@ -105,10 +105,10 @@ public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() var host = new IsIdentifiableHost(options, false); host.Start(); - tester.SendMessage(options.IsIdentifiableOptions, new ExtractFileStatusMessage + tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage { DicomFilePath = "yay.dcm", - AnonymisedFileName = testDcm.FullName, + OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", From 12116069b63d2f32aa4bda9509d60da88915a1d8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 18:21:09 +0100 Subject: [PATCH 062/138] Support identifiable extractions in CohortPackager --- CHANGELOG.md | 5 +- ...ctFileStatus.cs => ExtractedFileStatus.cs} | 18 +-- .../Extraction/ExtractedFileStatusMessage.cs | 10 +- ...cs => ExtractedFileVerificationMessage.cs} | 13 +- .../ExtractJobStorage/ExtractJobInfo.cs | 16 ++- .../ExtractJobStorage/ExtractJobStore.cs | 21 ++-- .../ExtractJobStorage/IExtractJobStore.cs | 9 +- .../MongoDB/MongoExtractJobInfoExtensions.cs | 3 +- .../MongoDB/MongoExtractJobStore.cs | 39 ++++-- .../MongoDB/ObjectModel/MongoFileStatusDoc.cs | 46 +++++-- .../Reporting/JobReporterBase.cs | 79 +++++++++--- .../AnonVerificationMessageConsumer.cs | 8 +- .../Execution/ExtractionFileCopier.cs | 4 +- .../Service/IsIdentifiableQueueConsumer.cs | 4 +- .../messages/ExtractedFileStatusMessage.java | 4 +- .../messaging/CTPAnonymiserConsumer.java | 10 +- ...leStatus.java => ExtractedFileStatus.java} | 13 +- .../test/execution/CTPAnonymiserHostTest.java | 6 +- .../Execution/CohortPackagerHostTest.cs | 112 ++++++++++++++++-- .../ExtractJobStorage/ExtractJobInfoTest.cs | 12 +- .../ExtractJobStorage/ExtractJobStoreTest.cs | 31 +++-- .../MongoExtractJobInfoExtensionsTest.cs | 4 +- .../MongoDB/MongoExtractJobStoreTest.cs | 14 ++- .../ObjectModel/MongoFileStatusDocTest.cs | 39 +++++- .../JobProcessing/ExtractJobWatcherTest.cs | 3 +- ...ReporterTest.cs => JobReporterBaseTest.cs} | 95 +++++++++++---- .../Execution/FileCopierTest.cs | 6 +- .../ServiceTests/IsIdentifiableHostTests.cs | 4 +- 28 files changed, 461 insertions(+), 167 deletions(-) rename src/common/Smi.Common/Messages/Extraction/{ExtractFileStatus.cs => ExtractedFileStatus.cs} (65%) rename src/common/Smi.Common/Messages/Extraction/{IsIdentifiableMessage.cs => ExtractedFileVerificationMessage.cs} (82%) rename src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/{ExtractFileStatus.java => ExtractedFileStatus.java} (74%) rename tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/{JobReporterTest.cs => JobReporterBaseTest.cs} (83%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b83c7db6..e7beb7a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -- Add SecurityCodeScan tool to build chain for .Net code +- Add SecurityCodeScan tool to build chain for .Net code +- Add identifiable extraction support + - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated + - [breaking] Changes to MongoDB extraction schema. Existing databases need to be updated ## [1.11.1] - 2020-08-12 diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs similarity index 65% rename from src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs rename to src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs index f0c60b463..da87ca49a 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs @@ -1,33 +1,27 @@ - -namespace Smi.Common.Messages.Extraction +namespace Smi.Common.Messages.Extraction { - // TODO(rkm 2020-03-07) Check what errors CTPAnonymiser can actually spit out here - public enum ExtractFileStatus + public enum ExtractedFileStatus { - Unknown = 0, - /// - /// The file has been anonymised successfully + /// Unused placeholder value /// - Anonymised, + Unused = 0, /// - /// The file could not be anonymised but will be retried later + /// The file has been anonymised successfully /// - ErrorWillRetry, + Anonymised, /// /// The file could not be anonymised and will not be retired /// ErrorWontRetry, - // TODO Java /// /// The source file could not be found under the given filesystem root /// FileMissing, - // TODO Java /// /// The source file was successfully copied to the destination /// diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs index feba42d4b..faa5cc29f 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs @@ -16,21 +16,21 @@ public class ExtractedFileStatusMessage : ExtractMessage, IFileReferenceMessage, public string DicomFilePath { get; set; } /// - /// The for this file + /// The for this file /// [JsonProperty(Required = Required.Always)] - public ExtractFileStatus Status { get; set; } + public ExtractedFileStatus Status { get; set; } /// /// Output file path, relative to the extraction directory. Only required if an output file has been produced /// - [JsonProperty(Required = Required.DisallowNull)] + [JsonProperty(Required = Required.AllowNull)] public string OutputFilePath { get; set; } /// /// Message required if Status is not 0 /// - [JsonProperty(Required = Required.DisallowNull)] + [JsonProperty(Required = Required.AllowNull)] public string StatusMessage { get; set; } @@ -44,7 +44,7 @@ public ExtractedFileStatusMessage(IExtractMessage request) public override string ToString() => $"{base.ToString()}," + $"DicomFilePath={DicomFilePath}," + - $"ExtractFileStatus={Status}," + + $"ExtractedFileStatus={Status}," + $"OutputFilePath={OutputFilePath}," + $"StatusMessage={StatusMessage}," + ""; diff --git a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs similarity index 82% rename from src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs rename to src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs index e08f8f8ea..e3e38a2ec 100644 --- a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs @@ -3,8 +3,7 @@ namespace Smi.Common.Messages.Extraction { - // TODO(rkm 2020-02-04) Rename to AnonVerificationMessage - public class IsIdentifiableMessage : ExtractMessage, IFileReferenceMessage, IEquatable + public class ExtractedFileVerificationMessage : ExtractMessage, IFileReferenceMessage, IEquatable { [JsonProperty(Required = Required.Always)] public bool IsIdentifiable { get; set; } @@ -25,9 +24,9 @@ public class IsIdentifiableMessage : ExtractMessage, IFileReferenceMessage, IEqu public string OutputFilePath { get; set; } [JsonConstructor] - public IsIdentifiableMessage() { } + public ExtractedFileVerificationMessage() { } - public IsIdentifiableMessage(Guid extractionJobIdentifier, string projectNumber, string extractionDirectory, DateTime jobSubmittedAt) + public ExtractedFileVerificationMessage(Guid extractionJobIdentifier, string projectNumber, string extractionDirectory, DateTime jobSubmittedAt) : this() { ExtractionJobIdentifier = extractionJobIdentifier; @@ -40,7 +39,7 @@ public IsIdentifiableMessage(Guid extractionJobIdentifier, string projectNumber, /// Creates a new instance copying all values from the given origin message /// /// - public IsIdentifiableMessage(ExtractedFileStatusMessage request) + public ExtractedFileVerificationMessage(ExtractedFileStatusMessage request) : this(request.ExtractionJobIdentifier, request.ProjectNumber, request.ExtractionDirectory, request.JobSubmittedAt) { @@ -50,7 +49,7 @@ public IsIdentifiableMessage(ExtractedFileStatusMessage request) #region Equality Members - public bool Equals(IsIdentifiableMessage other) + public bool Equals(ExtractedFileVerificationMessage other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; @@ -62,7 +61,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((IsIdentifiableMessage)obj); + return Equals((ExtractedFileVerificationMessage)obj); } public override int GetHashCode() diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs index af72c65e9..ae10d0919 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs @@ -1,6 +1,6 @@ +using JetBrains.Annotations; using System; using System.Text; -using JetBrains.Annotations; namespace Microservices.CohortPackager.Execution.ExtractJobStorage { @@ -53,6 +53,9 @@ public class ExtractJobInfo : IEquatable /// public ExtractJobStatus JobStatus { get; } + public bool IsIdentifiableExtraction { get; } + + public ExtractJobInfo( Guid extractionJobIdentifier, DateTime jobSubmittedAt, @@ -61,7 +64,8 @@ public ExtractJobInfo( [NotNull] string keyTag, uint keyValueCount, [CanBeNull] string extractionModality, - ExtractJobStatus jobStatus) + ExtractJobStatus jobStatus, + bool isIdentifiableExtraction) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentNullException(nameof(extractionJobIdentifier)); JobSubmittedAt = (jobSubmittedAt != default(DateTime)) ? jobSubmittedAt : throw new ArgumentNullException(nameof(jobSubmittedAt)); @@ -72,6 +76,7 @@ public ExtractJobInfo( if (extractionModality != null) ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); JobStatus = (jobStatus != default(ExtractJobStatus)) ? jobStatus : throw new ArgumentException(nameof(jobStatus)); + IsIdentifiableExtraction = isIdentifiableExtraction; } public override string ToString() @@ -83,6 +88,7 @@ public override string ToString() sb.AppendLine("KeyTag: " + KeyTag); sb.AppendLine("KeyCount: " + KeyValueCount); sb.AppendLine("ExtractionModality: " + ExtractionModality); + sb.AppendLine("IsIdentifiableExtraction: " + IsIdentifiableExtraction); return sb.ToString(); } @@ -99,6 +105,7 @@ public bool Equals(ExtractJobInfo other) KeyTag == other.KeyTag && KeyValueCount == other.KeyValueCount && ExtractionModality == other.ExtractionModality && + IsIdentifiableExtraction == other.IsIdentifiableExtraction && JobStatus == other.JobStatus; } @@ -123,9 +130,10 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ KeyTag.GetHashCode(); - hashCode = (hashCode * 397) ^ (int)KeyValueCount; + hashCode = (hashCode * 397) ^ (int) KeyValueCount; hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (int)JobStatus; + hashCode = (hashCode * 397) ^ (int) JobStatus; + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); return hashCode; } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index 51038ea1e..38011f4ab 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -43,16 +43,16 @@ public void PersistMessageToStore( [NotNull] ExtractedFileStatusMessage message, [NotNull] IMessageHeader header) { - if (message.Status == ExtractFileStatus.Unknown) - throw new ApplicationException("ExtractFileStatus was unknown"); - if (message.Status == ExtractFileStatus.Anonymised) + if (message.Status == ExtractedFileStatus.Unused) + throw new ApplicationException("ExtractedFileStatus was the default unused value"); + if (message.Status == ExtractedFileStatus.Anonymised) throw new ApplicationException("Received an anonymisation successful message from the failure queue"); PersistMessageToStoreImpl(message, header); } public void PersistMessageToStore( - [NotNull] IsIdentifiableMessage message, + [NotNull] ExtractedFileVerificationMessage message, [NotNull] IMessageHeader header) { if (string.IsNullOrWhiteSpace(message.OutputFilePath)) @@ -125,10 +125,18 @@ public IEnumerable> GetCompletedJobVerificationFailures(Gu return GetCompletedJobVerificationFailuresImpl(jobId); } + public IEnumerable GetCompletedJobMissingFileList(Guid jobId) + { + if (jobId == default) + throw new ArgumentNullException(nameof(jobId)); + + return GetCompletedJobMissingFileListImpl(jobId); + } + protected abstract void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header); protected abstract void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header); protected abstract void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header); - protected abstract void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header); + protected abstract void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header); protected abstract List GetReadyJobsImpl(Guid specificJobId = new Guid()); protected abstract void CompleteJobImpl(Guid jobId); protected abstract void MarkJobFailedImpl(Guid jobId, Exception e); @@ -136,7 +144,6 @@ public IEnumerable> GetCompletedJobVerificationFailures(Gu protected abstract IEnumerable>> GetCompletedJobRejectionsImpl(Guid jobId); protected abstract IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId); protected abstract IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId); - - + protected abstract IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs index 970558130..8854a16ca 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs @@ -37,7 +37,7 @@ public interface IExtractJobStore /// /// /// - void PersistMessageToStore(IsIdentifiableMessage anonVerificationMessage, IMessageHeader header); + void PersistMessageToStore(ExtractedFileVerificationMessage anonVerificationMessage, IMessageHeader header); /// /// Returns a list of all jobs which are ready for final checks @@ -86,5 +86,12 @@ public interface IExtractJobStore /// /// IEnumerable> GetCompletedJobVerificationFailures(Guid jobId); + + /// + /// Returns the full list of files that were requested but could not be produced, and a reason for each + /// + /// + /// + IEnumerable GetCompletedJobMissingFileList(Guid jobId); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs index b0c1574e4..a93c8165d 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs @@ -13,6 +13,7 @@ public static ExtractJobInfo ToExtractJobInfo(this MongoExtractJobDoc mongoExtra mongoExtractJobDoc.KeyTag, mongoExtractJobDoc.KeyCount, mongoExtractJobDoc.ExtractionModality, - mongoExtractJobDoc.JobStatus); + mongoExtractJobDoc.JobStatus, + mongoExtractJobDoc.IsIdentifiableExtraction); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs index b9160f322..3d9413fa1 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs @@ -77,9 +77,11 @@ protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage mes var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), + message.DicomFilePath, message.OutputFilePath, wasAnonymised: false, isIdentifiable: true, + message.Status, statusMessage: message.StatusMessage); _database @@ -87,16 +89,18 @@ protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage mes .InsertOne(newStatus); } - protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header) + protected override void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header) { if (InCompletedJobCollection(message.ExtractionJobIdentifier)) - throw new ApplicationException("Received an IsIdentifiableMessage for a job that is already completed"); - + throw new ApplicationException("Received an ExtractedFileVerificationMessage for a job that is already completed"); + var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), + message.DicomFilePath, message.OutputFilePath, wasAnonymised: true, isIdentifiable: message.IsIdentifiable, + ExtractedFileStatus.Anonymised, statusMessage: message.Report); _database @@ -322,26 +326,31 @@ protected override IEnumerable>> GetComple protected override IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId) { // NOTE(rkm 2020-03-16) Files which failed anonymisation should have statuses where WasAnonymised=false and IsIdentifiable=true - FilterDefinition filter = Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); filter &= Builders.Filter.Eq(x => x.WasAnonymised, false); filter &= Builders.Filter.Eq(x => x.IsIdentifiable, true); - - IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); - while (cursor.MoveNext()) - foreach (MongoFileStatusDoc doc in cursor.Current) - yield return new Tuple(doc.AnonymisedFileName, doc.StatusMessage); + return CompletedStatusDocsForFilter(filter); } protected override IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId) { - FilterDefinition filter = Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); filter &= Builders.Filter.Eq(x => x.WasAnonymised, true); filter &= Builders.Filter.Eq(x => x.IsIdentifiable, true); + return CompletedStatusDocsForFilter(filter); + } + protected override IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId) + { + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + filter &= Builders.Filter.Eq(x => x.ExtractedFileStatus, ExtractedFileStatus.FileMissing); IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); while (cursor.MoveNext()) foreach (MongoFileStatusDoc doc in cursor.Current) - yield return new Tuple(doc.AnonymisedFileName, doc.StatusMessage); + yield return doc.DicomFilePath; } #region Helper Methods @@ -369,6 +378,14 @@ private bool InCompletedJobCollection(Guid extractionJobIdentifier) .SingleOrDefault() != null; } + private IEnumerable> CompletedStatusDocsForFilter(FilterDefinition filter) + { + IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); + while (cursor.MoveNext()) + foreach (MongoFileStatusDoc doc in cursor.Current) + yield return new Tuple(doc.OutputFileName, doc.StatusMessage); + } + #endregion } } \ No newline at end of file diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs index 69d2d3f77..fc63c2cc0 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs @@ -1,7 +1,8 @@ -using System; -using JetBrains.Annotations; +using JetBrains.Annotations; +using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; - +using Smi.Common.Messages.Extraction; +using System; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel { @@ -15,9 +16,13 @@ public class MongoFileStatusDoc [NotNull] public MongoExtractionMessageHeaderDoc Header { get; set; } - [BsonElement("anonymisedFileName")] + [BsonElement("dicomFilePath")] [NotNull] - public string AnonymisedFileName { get; set; } + public string DicomFilePath { get; set; } + + [BsonElement("outputFileName")] + [CanBeNull] + public string OutputFileName { get; set; } [BsonElement("wasAnonymised")] public bool WasAnonymised { get; set; } @@ -25,22 +30,35 @@ public class MongoFileStatusDoc [BsonElement("isIdentifiable")] public bool IsIdentifiable { get; set; } + [BsonElement("extractedFileStatus")] + [BsonRepresentation(BsonType.String)] + public ExtractedFileStatus ExtractedFileStatus { get; set; } + + /// + /// Should only be null for identifiable extractions where the file was successfully copied. Otherwise will be the failure reason from CTP or the report content from the IsIdentifiable verification + /// [BsonElement("statusMessage")] - [NotNull] // NOTE(rkm 2020-02-27) Will be the failure reason from an ExtractFileStatusMessage, or the report content from an IsIdentifiableMessage + [CanBeNull] public string StatusMessage { get; set; } public MongoFileStatusDoc( [NotNull] MongoExtractionMessageHeaderDoc header, - [NotNull] string anonymisedFileName, + [NotNull] string dicomFilePath, + [CanBeNull] string outputFileName, bool wasAnonymised, bool isIdentifiable, - [NotNull] string statusMessage) + ExtractedFileStatus extractedFileStatus, + [CanBeNull] string statusMessage) { Header = header ?? throw new ArgumentNullException(nameof(header)); - AnonymisedFileName = anonymisedFileName; + DicomFilePath = dicomFilePath ?? throw new ArgumentNullException(nameof(dicomFilePath)); + OutputFileName = outputFileName; WasAnonymised = wasAnonymised; IsIdentifiable = isIdentifiable; - StatusMessage = (!string.IsNullOrWhiteSpace(statusMessage)) ? statusMessage : throw new ArgumentNullException(nameof(statusMessage)); + ExtractedFileStatus = (extractedFileStatus != ExtractedFileStatus.Unused) ? extractedFileStatus : throw new ArgumentException(nameof(extractedFileStatus)); + StatusMessage = statusMessage; + if (!IsIdentifiable && string.IsNullOrWhiteSpace(statusMessage)) + throw new ArgumentNullException(nameof(statusMessage)); } #region Equality Methods @@ -48,9 +66,11 @@ public MongoFileStatusDoc( protected bool Equals(MongoFileStatusDoc other) { return Equals(Header, other.Header) && - AnonymisedFileName == other.AnonymisedFileName && + DicomFilePath == other.DicomFilePath && + OutputFileName == other.OutputFileName && WasAnonymised == other.WasAnonymised && IsIdentifiable == other.IsIdentifiable && + ExtractedFileStatus == other.ExtractedFileStatus && StatusMessage == other.StatusMessage; } @@ -71,9 +91,11 @@ public override int GetHashCode() unchecked { int hashCode = (Header.GetHashCode()); - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFileName != null ? OutputFileName.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (WasAnonymised.GetHashCode()); hashCode = (hashCode * 397) ^ (IsIdentifiable.GetHashCode()); + hashCode = (hashCode * 397) ^ (ExtractedFileStatus.GetHashCode()); hashCode = (hashCode * 397) ^ (StatusMessage.GetHashCode()); return hashCode; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs index 11944b15b..fbfd6313b 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs @@ -38,22 +38,36 @@ public void CreateReport(Guid jobId) streamWriter.WriteLine(line); streamWriter.WriteLine(); - streamWriter.WriteLine("## Verification failures"); - streamWriter.WriteLine(); - WriteJobVerificationFailures(streamWriter, _jobStore.GetCompletedJobVerificationFailures(jobInfo.ExtractionJobIdentifier)); - streamWriter.WriteLine(); + if (jobInfo.IsIdentifiableExtraction) + { + streamWriter.WriteLine("## Missing file list"); + streamWriter.WriteLine(); + WriteJobMissingFileList(streamWriter, _jobStore.GetCompletedJobMissingFileList(jobInfo.ExtractionJobIdentifier)); + streamWriter.WriteLine(); + } + else + { - streamWriter.WriteLine("## Rejected files"); - streamWriter.WriteLine(); - foreach (Tuple> rejection in _jobStore.GetCompletedJobRejections(jobInfo.ExtractionJobIdentifier)) - WriteJobRejections(streamWriter, rejection); - streamWriter.WriteLine(); + streamWriter.WriteLine("## Verification failures"); + streamWriter.WriteLine(); + WriteJobVerificationFailures(streamWriter, + _jobStore.GetCompletedJobVerificationFailures(jobInfo.ExtractionJobIdentifier)); + streamWriter.WriteLine(); - streamWriter.WriteLine("## Anonymisation failures"); - streamWriter.WriteLine(); - foreach ((string expectedAnonFile, string failureReason) in _jobStore.GetCompletedJobAnonymisationFailures(jobInfo.ExtractionJobIdentifier)) - WriteAnonFailure(streamWriter, expectedAnonFile, failureReason); - streamWriter.WriteLine(); + streamWriter.WriteLine("## Rejected files"); + streamWriter.WriteLine(); + foreach (Tuple> rejection in _jobStore.GetCompletedJobRejections( + jobInfo.ExtractionJobIdentifier)) + WriteJobRejections(streamWriter, rejection); + streamWriter.WriteLine(); + + streamWriter.WriteLine("## Anonymisation failures"); + streamWriter.WriteLine(); + foreach ((string expectedAnonFile, string failureReason) in _jobStore + .GetCompletedJobAnonymisationFailures(jobInfo.ExtractionJobIdentifier)) + WriteAnonFailure(streamWriter, expectedAnonFile, failureReason); + streamWriter.WriteLine(); + } streamWriter.WriteLine("--- end of report ---"); @@ -68,7 +82,9 @@ public void CreateReport(Guid jobId) protected abstract void FinishReport(Stream stream); private static IEnumerable JobHeader(ExtractJobInfo jobInfo) - => new[] + { + string identExtraction = jobInfo.IsIdentifiableExtraction ? "Yes" : "No"; + var header = new List { $"# SMI file extraction report for {jobInfo.ProjectNumber}", "", @@ -78,13 +94,32 @@ private static IEnumerable JobHeader(ExtractJobInfo jobInfo) $"- Extraction tag: {jobInfo.KeyTag}", $"- Extraction modality: {jobInfo.ExtractionModality ?? "Unspecified"}", $"- Requested identifier count: {jobInfo.KeyValueCount}", + $"- Identifiable extraction: {identExtraction}", "", - "Report contents:", - "- Verification failures", - "- Rejected failures", - "- Anonymisation failures", }; + if (jobInfo.IsIdentifiableExtraction) + { + header.AddRange(new List + { + "Report contents:", + "- Missing file list (files which were selected from an input ID but could not be found)", + }); + } + else + { + header.AddRange(new List + { + "Report contents:", + "- Verification failures", + "- Rejected failures", + "- Anonymisation failures", + }); + } + + return header; + } + private static void WriteJobRejections(TextWriter streamWriter, Tuple> rejection) { (string rejectionKey, Dictionary rejectionItems) = rejection; @@ -165,6 +200,12 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume streamWriter.Write(sb); } + private static void WriteJobMissingFileList(TextWriter streamWriter, IEnumerable missingFiles) + { + foreach (string file in missingFiles) + streamWriter.WriteLine($"- {file}"); + } + protected abstract void ReleaseUnmanagedResources(); public abstract void Dispose(); ~JobReporterBase() => ReleaseUnmanagedResources(); diff --git a/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs b/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs index 1945e13cb..d353cd842 100644 --- a/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs +++ b/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs @@ -12,9 +12,9 @@ namespace Microservices.CohortPackager.Messaging { /// - /// Consumer for (s) + /// Consumer for (s) /// - public class AnonVerificationMessageConsumer : Consumer + public class AnonVerificationMessageConsumer : Consumer { private readonly IExtractJobStore _store; @@ -25,7 +25,7 @@ public AnonVerificationMessageConsumer(IExtractJobStore store) } - protected override void ProcessMessageImpl(IMessageHeader header, IsIdentifiableMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileVerificationMessage message, ulong tag) { try { @@ -45,7 +45,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, IsIdentifiable catch (ApplicationException e) { // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage - ErrorAndNack(header, tag, "Error while processing IsIdentifiableMessage", e); + ErrorAndNack(header, tag, "Error while processing ExtractedFileVerificationMessage", e); return; } diff --git a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs index e8f73bb84..f423579ed 100644 --- a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -49,7 +49,7 @@ public void ProcessMessage( statusMessage = new ExtractedFileStatusMessage(message) { DicomFilePath = message.DicomFilePath, - Status = ExtractFileStatus.FileMissing, + Status = ExtractedFileStatus.FileMissing, StatusMessage = $"Could not find '{fullSrc}'" }; _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); @@ -74,7 +74,7 @@ public void ProcessMessage( statusMessage = new ExtractedFileStatusMessage(message) { DicomFilePath = message.DicomFilePath, - Status = ExtractFileStatus.Copied, + Status = ExtractedFileStatus.Copied, OutputFilePath = message.OutputPath, }; _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); diff --git a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs index 44b13fa2b..c88f282c4 100644 --- a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs +++ b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs @@ -34,7 +34,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileS try { // We should only ever receive messages regarding anonymised images - if (message.Status != ExtractFileStatus.Anonymised) + if (message.Status != ExtractedFileStatus.Anonymised) throw new ApplicationException($"Received a message with anonymised status of {message.Status}"); var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputFilePath) ); @@ -58,7 +58,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileS return; } - _producer.SendMessage(new IsIdentifiableMessage(message) + _producer.SendMessage(new ExtractedFileVerificationMessage(message) { IsIdentifiable = ! isClean, Report = JsonConvert.SerializeObject(resultObject) diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java index d9c48c330..1a07d3e2d 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java @@ -3,7 +3,7 @@ import org.smi.common.messageSerialization.JsonDeserializerWithOptions.FieldRequired; import org.smi.common.messages.ExtractMessage; import org.smi.common.messages.IMessage; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; /** * Message indicating the path to an anonymised file @@ -16,7 +16,7 @@ public class ExtractedFileStatusMessage extends ExtractMessage implements IMessa public String OutputFilePath; @FieldRequired - public ExtractFileStatus Status; + public ExtractedFileStatus Status; public String StatusMessage; diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java index 936ae78d6..bd138023b 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java @@ -14,7 +14,7 @@ import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; import org.smi.ctpanonymiser.util.CtpAnonymisationStatus; import org.smi.common.options.GlobalOptions; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; import java.io.File; import java.io.FileNotFoundException; @@ -98,7 +98,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope } statusMessage.StatusMessage = msg; - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.FileMissing; _statusMessageProducer.SendMessage(statusMessage, _routingKey_failure, header); @@ -134,7 +134,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope _logger.error(msg); statusMessage.StatusMessage = msg; - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.FileMissing; _statusMessageProducer.SendMessage(statusMessage, _routingKey_failure, header); @@ -156,13 +156,13 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope if (status == CtpAnonymisationStatus.Anonymised) { statusMessage.OutputFilePath = extractFileMessage.OutputPath; - statusMessage.Status = ExtractFileStatus.Anonymised; + statusMessage.Status = ExtractedFileStatus.Anonymised; routingKey = _routingKey_success; } else { statusMessage.StatusMessage = _anonTool.getLastStatus(); - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.ErrorWontRetry; routingKey = _routingKey_failure; } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java similarity index 74% rename from src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java rename to src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java index ade2389ec..2c4bb15ad 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java @@ -1,20 +1,17 @@ package org.smi.ctpanonymiser.util; -public enum ExtractFileStatus { - - Unknown, +public enum ExtractedFileStatus { + /** + * Unused placeholder value + */ + Unused, /** * The file has been anonymised successfully */ Anonymised, - /** - * The file could not be anonymised but will be retried later - */ - ErrorWillRetry, - /** * The file could not be anonymised and will not be retired */ diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index d93a228c2..2547b2a2e 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -17,7 +17,7 @@ import org.smi.ctpanonymiser.execution.CTPAnonymiserHost; import org.smi.ctpanonymiser.messages.ExtractFileMessage; import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; import java.io.File; import java.nio.file.Paths; @@ -192,7 +192,7 @@ public void testBasicAnonymise_Success() throws InterruptedException { assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.OutputFilePath); assertEquals("Project numbers do not match", exMessage.ProjectNumber, recvd.ProjectNumber); - assertEquals(ExtractFileStatus.Anonymised, recvd.Status); + assertEquals(ExtractedFileStatus.Anonymised, recvd.Status); } else { fail("Did not receive message"); } @@ -241,7 +241,7 @@ public void testBasicAnonymise_Failure() throws InterruptedException { _logger.info("\n" + recvd.toString()); assertEquals("FilePaths do not match", null, recvd.OutputFilePath); - assertEquals(ExtractFileStatus.ErrorWontRetry, recvd.Status); + assertEquals(ExtractedFileStatus.ErrorWontRetry, recvd.Status); } else { fail("Did not receive message"); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs index 170404b1e..398affd16 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Threading; using Microservices.CohortPackager.Execution; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.JobProcessing.Notifying; @@ -13,6 +10,9 @@ using Smi.Common.MongoDB; using Smi.Common.Options; using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Threading; namespace Microservices.CohortPackager.Tests.Execution { @@ -47,11 +47,8 @@ public void TearDown() { } private class TestReporter : IJobReporter { - public string Report { get; set; } - public void CreateReport(Guid jobId) - { - Report = $"Report for {jobId}"; - } + public bool reportCreated; + public void CreateReport(Guid jobId) => reportCreated = true; } private class TestLoggingNotifier : IJobCompleteNotifier @@ -102,11 +99,11 @@ public void Test_CohortPackagerHost_HappyPath() ProjectNumber = "testProj1", ExtractionJobIdentifier = jobId, ExtractionDirectory = "test", - Status = ExtractFileStatus.ErrorWontRetry, + Status = ExtractedFileStatus.ErrorWontRetry, StatusMessage = "Couldn't anonymise", DicomFilePath = "study-1-orig-1.dcm", }; - var testIsIdentifiableMessage = new IsIdentifiableMessage + var testIsIdentifiableMessage = new ExtractedFileVerificationMessage { JobSubmittedAt = DateTime.UtcNow, OutputFilePath = "study-1-anon-2.dcm", @@ -151,6 +148,101 @@ public void Test_CohortPackagerHost_HappyPath() host.Stop("Test end"); Assert.True(notifier.JobCompleted && timeoutSecs >= 0); + Assert.True(reporter.reportCreated); + } + } + + [Test] + public void Test_CohortPackagerHost_IdentifiableExtraction() + { + Guid jobId = Guid.NewGuid(); + var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage + { + ExtractionModality = "MR", + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + KeyTag = "StudyInstanceUID", + KeyValueCount = 1, + IsIdentifiableExtraction = true, + }; + var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage + { + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + ExtractFileMessagesDispatched = new JsonCompatibleDictionary + { + { new MessageHeader(), "out1.dcm" }, + { new MessageHeader(), "out2.dcm" }, + }, + RejectionReasons = new Dictionary + { + {"rejected - blah", 1 }, + }, + KeyValue = "study-1", + IsIdentifiableExtraction = true, + }; + var testExtractFileStatusMessage1 = new ExtractedFileStatusMessage + { + JobSubmittedAt = DateTime.UtcNow, + OutputFilePath = "src.dcm", + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + Status = ExtractedFileStatus.Copied, + StatusMessage = null, + DicomFilePath = "study-1-orig-1.dcm", + IsIdentifiableExtraction = true, + }; + var testExtractFileStatusMessage2 = new ExtractedFileStatusMessage + { + JobSubmittedAt = DateTime.UtcNow, + OutputFilePath = "src_missing.dcm", + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + Status = ExtractedFileStatus.FileMissing, + StatusMessage = null, + DicomFilePath = "study-1-orig-2.dcm", + IsIdentifiableExtraction = true, + }; + + GlobalOptions globals = GlobalOptions.Load(); + globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; + + MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); + client.DropDatabase(globals.MongoDatabases.ExtractionStoreOptions.DatabaseName); + + using (var tester = new MicroserviceTester( + globals.RabbitOptions, + globals.CohortPackagerOptions.ExtractRequestInfoOptions, + globals.CohortPackagerOptions.FileCollectionInfoOptions, + globals.CohortPackagerOptions.NoVerifyStatusOptions, + globals.CohortPackagerOptions.VerificationStatusOptions)) + { + tester.SendMessage(globals.CohortPackagerOptions.ExtractRequestInfoOptions, new MessageHeader(), testExtractionRequestInfoMessage); + tester.SendMessage(globals.CohortPackagerOptions.FileCollectionInfoOptions, new MessageHeader(), testExtractFileCollectionInfoMessage); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage1); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage2); + + var reporter = new TestReporter(); + var notifier = new TestLoggingNotifier(); + var host = new CohortPackagerHost(globals, reporter, notifier, null, false); + host.Start(); + + var timeoutSecs = 30; + while (!notifier.JobCompleted && timeoutSecs > 0) + { + --timeoutSecs; + Thread.Sleep(TimeSpan.FromSeconds(1)); + } + + host.Stop("Test end"); + Assert.True(notifier.JobCompleted && timeoutSecs >= 0); + Assert.True(reporter.reportCreated); } } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs index 16a6bd0b0..37fbca4cd 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs @@ -48,7 +48,8 @@ public void TestExtractJobInfo_Equality() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + true); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -57,7 +58,8 @@ public void TestExtractJobInfo_Equality() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + true); Assert.AreEqual(info1, info2); } @@ -74,7 +76,8 @@ public void TestExtractJobInfo_GetHashCode() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + true); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -83,7 +86,8 @@ public void TestExtractJobInfo_GetHashCode() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + true); Assert.AreEqual(info1.GetHashCode(), info2.GetHashCode()); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs index ab8bfdfeb..1d403abcf 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs @@ -1,10 +1,10 @@ -using System; -using System.Collections.Generic; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using NUnit.Framework; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Collections.Generic; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage @@ -35,7 +35,7 @@ private class TestExtractJobStore : ExtractJobStore protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header) { } protected override void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header) => throw new NotImplementedException(); protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header) { } - protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header) { } + protected override void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header) { } protected override List GetReadyJobsImpl(Guid specificJobId = new Guid()) => throw new NotImplementedException(); protected override void CompleteJobImpl(Guid jobId) { } protected override void MarkJobFailedImpl(Guid jobId, Exception e) { } @@ -43,6 +43,7 @@ protected override void MarkJobFailedImpl(Guid jobId, Exception e) { } protected override IEnumerable>> GetCompletedJobRejectionsImpl(Guid jobId) => throw new NotImplementedException(); protected override IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId) => throw new NotImplementedException(); protected override IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId) => throw new NotImplementedException(); + protected override IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId) => new[] { "missing" }; } #endregion @@ -80,13 +81,13 @@ public void TestPersistMessageToStore_ExtractFileStatusMessage() var message = new ExtractedFileStatusMessage(); var header = new MessageHeader(); - message.Status = ExtractFileStatus.Unknown; + message.Status = ExtractedFileStatus.Unused; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); - message.Status = ExtractFileStatus.Anonymised; + message.Status = ExtractedFileStatus.Anonymised; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); - message.Status = ExtractFileStatus.ErrorWontRetry; + message.Status = ExtractedFileStatus.ErrorWontRetry; testExtractJobStore.PersistMessageToStore(message, header); } @@ -98,18 +99,18 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() var header = new MessageHeader(); // Must have AnonymisedFileName - var message = new IsIdentifiableMessage(); + var message = new ExtractedFileVerificationMessage(); message.OutputFilePath = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report shouldn't be an empty string or null - message = new IsIdentifiableMessage(); + message = new ExtractedFileVerificationMessage(); message.OutputFilePath = "anon.dcm"; message.Report = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report needs to contain content if marked as IsIdentifiable - message = new IsIdentifiableMessage(); + message = new ExtractedFileVerificationMessage(); message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = true; message.Report = "[]"; @@ -119,7 +120,7 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() testExtractJobStore.PersistMessageToStore(message, header); // Report can be empty if not marked as IsIdentifiable - message = new IsIdentifiableMessage(); + message = new ExtractedFileVerificationMessage(); message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = false; message.Report = "[]"; @@ -147,6 +148,14 @@ public void TestMarkJobFailed() store.MarkJobFailed(Guid.NewGuid(), new Exception()); } + [Test] + public void Test_GetCompletedJobMissingFileList() + { + var store = new TestExtractJobStore(); + Assert.Throws(() => store.GetCompletedJobMissingFileList(default)); + Assert.AreEqual(new[] { "missing" }, store.GetCompletedJobMissingFileList(Guid.NewGuid())); + } + #endregion } } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs index 25764fed7..cef7f9de2 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs @@ -59,6 +59,7 @@ public void TestToExtractJobInfo() ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, + IsIdentifiableExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); @@ -72,7 +73,8 @@ public void TestToExtractJobInfo() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + true); Assert.AreEqual(expected, extractJobInfo); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs index ea9a27e93..ec302168a 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs @@ -17,6 +17,7 @@ using System.Linq; using System.Threading; + namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB { [TestFixture] @@ -364,7 +365,7 @@ public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() { OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), - Status = ExtractFileStatus.ErrorWontRetry, + Status = ExtractedFileStatus.ErrorWontRetry, ProjectNumber = "1234", ExtractionJobIdentifier = jobId, ExtractionDirectory = "1234/test", @@ -375,16 +376,17 @@ public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() store.PersistMessageToStore(testExtractFileStatusMessage, header); - Dictionary docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents; Assert.AreEqual(docs.Count, 1); MongoFileStatusDoc statusDoc = docs.Values.ToList()[0]; var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), + "original.dcm", "anon.dcm", false, true, + ExtractedFileStatus.ErrorWontRetry, "Could not anonymise"); Assert.True(statusDoc.Equals(expected)); @@ -397,7 +399,7 @@ public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); - var testIsIdentifiableMessage = new IsIdentifiableMessage + var testIsIdentifiableMessage = new ExtractedFileVerificationMessage { OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), @@ -418,9 +420,11 @@ public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), + "original.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "[]"); Assert.True(statusDoc.Equals(expected)); @@ -455,9 +459,11 @@ public void TestGetReadJobsImpl() ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), + "input.dcm", "anon1.dcm", true, false, + ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); @@ -531,9 +537,11 @@ public void TestCompleteJobImpl() ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), + "input.dcm", "anon1.dcm", true, false, + ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs index 862a03a46..311692e8d 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs @@ -1,9 +1,11 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; using NUnit.Framework; +using Smi.Common.Helpers; using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -39,6 +41,29 @@ public void TearDown() { } #region Tests + [Test] + public void Test_MongoFileStatusDoc_IsIdentifiable_StatusMessage() + { + Assert.Throws(() => + new MongoFileStatusDoc( + MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()), + "input.dcm", + "anon.dcm", + true, + false, + ExtractedFileStatus.Anonymised, + null)); + Assert.DoesNotThrow(() => + new MongoFileStatusDoc( + MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()), + "input.dcm", + "anon.dcm", + true, + true, + ExtractedFileStatus.Anonymised, + null)); + } + [Test] public void TestMongoFileStatusDoc_SettersAvailable() { @@ -52,16 +77,20 @@ public void TestMongoFileStatusDoc_Equality() Guid guid = Guid.NewGuid(); var doc1 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); var doc2 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); Assert.AreEqual(doc1, doc2); @@ -73,16 +102,20 @@ public void TestMongoFileStatusDoc_GetHashCode() Guid guid = Guid.NewGuid(); var doc1 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); var doc2 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs index f81d19686..e45ed7b39 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs @@ -70,7 +70,8 @@ public void TestProcessJobs() "KeyTag", 123, null, - ExtractJobStatus.ReadyForChecks + ExtractJobStatus.ReadyForChecks, + true ); var opts = new CohortPackagerOptions { JobWatcherTimeoutInSeconds = 123 }; diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs similarity index 83% rename from tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs rename to tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs index 7871d4d78..6ab5b431e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs @@ -12,7 +12,7 @@ namespace Microservices.CohortPackager.Tests.Execution.JobProcessing.Reporting { [TestFixture] - public class JobReporterTest + public class JobReporterBaseTest { #region Fixture Methods @@ -42,13 +42,9 @@ private class TestJobReporter : JobReporterBase public bool Disposed { get; set; } - public TestJobReporter(IExtractJobStore jobStore) - : base(jobStore, null) { } + public TestJobReporter(IExtractJobStore jobStore) : base(jobStore, null) { } - protected override Stream GetStream(Guid jobId) - { - return new MemoryStream(); - } + protected override Stream GetStream(Guid jobId) => new MemoryStream(); protected override void FinishReport(Stream stream) { @@ -58,17 +54,8 @@ protected override void FinishReport(Stream stream) } } - - protected override void ReleaseUnmanagedResources() - { - Disposed = true; - } - - - public override void Dispose() - { - ReleaseUnmanagedResources(); - } + protected override void ReleaseUnmanagedResources() => Disposed = true; + public override void Dispose() => ReleaseUnmanagedResources(); } [Test] @@ -84,7 +71,8 @@ public void Test_JobReporterBase_CreateReport_Empty() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + false); var mockJobStore = new Mock(MockBehavior.Strict); mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); @@ -107,6 +95,7 @@ public void Test_JobReporterBase_CreateReport_Empty() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No Report contents: - Verification failures @@ -146,7 +135,8 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + false); var rejections = new List>> { @@ -201,6 +191,7 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No Report contents: - Verification failures @@ -252,7 +243,8 @@ public void Test_JobReporterBase_WriteJobVerificationFailures_JsonException() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + false); var verificationFailures = new List> { @@ -287,7 +279,8 @@ public void Test_JobReporterBase_CreateReport_AggregateData() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + false); var verificationFailures = new List> { @@ -370,6 +363,7 @@ public void Test_JobReporterBase_CreateReport_AggregateData() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No Report contents: - Verification failures @@ -414,7 +408,62 @@ public void Test_JobReporterBase_CreateReport_AggregateData() TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); Assert.True(reporter.Disposed); } - } + [Test] + public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() + { + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed, + true); + + var missingFiles = new List + { + "missing.dcm", + }; + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobMissingFileList(It.IsAny())).Returns(missingFiles); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 +- Identifiable extraction: Yes + +Report contents: +- Missing file list (files which were selected from an input ID but could not be found) + +## Missing file list + +- missing.dcm + +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } + + } #endregion } diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs index ebf4e3fb7..819ee4cfb 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -92,7 +92,7 @@ public void Test_FileCopier_HappyPath() var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, - Status = ExtractFileStatus.Copied, + Status = ExtractedFileStatus.Copied, OutputFilePath = _requestMessage.OutputPath, }; Assert.AreEqual(expectedStatusMessage, sentStatusMessage); @@ -127,7 +127,7 @@ public void Test_FileCopier_MissingFile_SendsMessage() var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, - Status = ExtractFileStatus.FileMissing, + Status = ExtractedFileStatus.FileMissing, OutputFilePath = null, StatusMessage = $"Could not find '{_mockFileSystem.Path.Combine(FileSystemRoot, "missing.dcm")}'" }; @@ -161,7 +161,7 @@ public void Test_FileCopier_ExistingOutputFile_IsOverwritten() var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) { DicomFilePath = _requestMessage.DicomFilePath, - Status = ExtractFileStatus.Copied, + Status = ExtractedFileStatus.Copied, OutputFilePath = _requestMessage.OutputPath, StatusMessage = null, }; diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs index 034cbd631..891602121 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs @@ -69,7 +69,7 @@ public void TestClassifierName_ValidClassifier() ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", - Status = ExtractFileStatus.Anonymised + Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); @@ -112,7 +112,7 @@ public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", - Status = ExtractFileStatus.Anonymised + Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); From f8321f2e0c8c107a7a089960d633464eb81a62c5 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 18:32:02 +0100 Subject: [PATCH 063/138] Ensure RejectReason specified if Reject=true --- .../Execution/RequestFulfillers/FakeFulfiller.cs | 1 + .../Execution/RequestFulfillers/QueryToExecuteResult.cs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs index 41a9cfa20..184a33718 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs @@ -7,6 +7,7 @@ namespace Microservices.CohortExtractor.Execution.RequestFulfillers { + // TODO(rkm 2020-08-27) What is this used for? public class FakeFulfiller : IExtractionRequestFulfiller { protected readonly Logger Logger; diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs index 3d7cae18e..77cf07884 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs @@ -1,5 +1,7 @@ +using JetBrains.Annotations; using System; + namespace Microservices.CohortExtractor.Execution.RequestFulfillers { public class QueryToExecuteResult : IEquatable @@ -20,6 +22,8 @@ public QueryToExecuteResult(string filePathValue, string studyTagValue, string s InstanceTagValue = instanceTagValue; Reject = rejection; RejectReason = rejectionReason; + if (Reject && string.IsNullOrWhiteSpace(RejectReason)) + throw new ArgumentException("RejectReason must be specified if Reject=true"); } public override string ToString() From dfec42d7e111bbb8798d0309792ef34766aea120 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 18:51:44 +0100 Subject: [PATCH 064/138] Fix Java test --- .../smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index 2547b2a2e..ff4acaba1 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -241,7 +241,7 @@ public void testBasicAnonymise_Failure() throws InterruptedException { _logger.info("\n" + recvd.toString()); assertEquals("FilePaths do not match", null, recvd.OutputFilePath); - assertEquals(ExtractedFileStatus.ErrorWontRetry, recvd.Status); + assertEquals(ExtractedFileStatus.FileMissing, recvd.Status); } else { fail("Did not receive message"); } From 04ac3d1d589464b480c38861526c0b7bffd6299f Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 27 Aug 2020 19:12:47 +0100 Subject: [PATCH 065/138] Tidy --- branch_todo.md | 7 ------- .../Messages/Extraction/ExtractedFileStatusMessage.cs | 1 - 2 files changed, 8 deletions(-) delete mode 100644 branch_todo.md diff --git a/branch_todo.md b/branch_todo.md deleted file mode 100644 index 74a94990f..000000000 --- a/branch_todo.md +++ /dev/null @@ -1,7 +0,0 @@ - -# Branch TODO - -- Update the extraction plan doc -- Ensure changes to message definitions are reflected in the Java code -- Update RMQ exchange names, queue names and binding keys - - Java currently has them as "success" or "failure", which doesn't make sense anymore. Change to "verify" and "noverify" diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs index faa5cc29f..0fd69f155 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs @@ -34,7 +34,6 @@ public class ExtractedFileStatusMessage : ExtractMessage, IFileReferenceMessage, public string StatusMessage { get; set; } - // TODO [JsonConstructor] public ExtractedFileStatusMessage() { } From 292117746c446e46ec0d04b876c10886ef27bee4 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 13:38:18 +0100 Subject: [PATCH 066/138] Add tests for FileCopierHost and CohortExtractor consumer --- data/microserviceConfigs/default.yaml | 7 +- .../Smi.Common/Events/FatalErrorEventArgs.cs | 11 +- .../Execution/ExtractionFileCopier.cs | 4 + .../Execution/FileCopierHost.cs | 16 ++- .../Smi.Common.Tests/MicroserviceTester.cs | 10 +- .../ExtractionRequestQueueConsumerTest.cs | 128 ++++++++++++++++-- .../Execution/FileCopierHostTest.cs | 72 +++++++++- 7 files changed, 216 insertions(+), 32 deletions(-) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index ab03a3f92..d7f004e3e 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -172,10 +172,9 @@ CTPAnonymiserOptions: FileCopierOptions: NoVerifyRoutingKey: noverify - CopyFileConsumerOptions: - QueueName: 'TEST.ExtractFileIdentQueue' - QoSPrefetchCount: 1 - AutoAck: false + QueueName: 'TEST.ExtractFileIdentQueue' + QoSPrefetchCount: 1 + AutoAck: false CopyStatusProducerOptions: ExchangeName: 'TEST.FileStatusExchange' MaxConfirmAttempts: 1 diff --git a/src/common/Smi.Common/Events/FatalErrorEventArgs.cs b/src/common/Smi.Common/Events/FatalErrorEventArgs.cs index 231ea6be9..593de7f47 100644 --- a/src/common/Smi.Common/Events/FatalErrorEventArgs.cs +++ b/src/common/Smi.Common/Events/FatalErrorEventArgs.cs @@ -6,7 +6,7 @@ namespace Smi.Common.Events { public class FatalErrorEventArgs : EventArgs { - public string Message { get; set; } + public string Message { get; } public Exception Exception { get; } @@ -21,5 +21,14 @@ public FatalErrorEventArgs(BasicReturnEventArgs ra) Message = string.Format("BasicReturnEventArgs: {0} - {1}. (Exchange: {2}, RoutingKey: {3})", ra.ReplyCode, ra.ReplyText, ra.Exchange, ra.RoutingKey); } + + public override string ToString() + { + return "" + + $"{base.ToString()}, " + + $"Message={Message}, " + + $"Exception={Exception}, " + + ""; + } } } diff --git a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs index f423579ed..945387343 100644 --- a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -4,6 +4,7 @@ using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; using Smi.Common.Options; +using System; using System.IO.Abstractions; @@ -32,6 +33,9 @@ public ExtractionFileCopier( _fileSystemRoot = fileSystemRoot; _fileSystem = fileSystem ?? new FileSystem(); + if (!_fileSystem.Directory.Exists(_fileSystemRoot)) + throw new ArgumentException($"Cannot find the specified fileSystemRoot: '{_fileSystemRoot}'"); + _logger = LogManager.GetLogger(GetType().Name); } diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs index 5eee5b62e..7315176a6 100644 --- a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs @@ -5,6 +5,8 @@ using Smi.Common.Options; using System; using System.IO; +using System.IO.Abstractions; + namespace Microservices.FileCopier.Execution { @@ -14,17 +16,19 @@ public class FileCopierHost : MicroserviceHost public FileCopierHost( [NotNull] GlobalOptions options, - bool loadSmiLogConfig = true) - : base(options, loadSmiLogConfig: loadSmiLogConfig) + [CanBeNull]IFileSystem fileSystem = null, + bool loadSmiLogConfig = true + ) + : base( + options, + loadSmiLogConfig: loadSmiLogConfig + ) { - if (!Directory.Exists(Globals.FileSystemOptions.FileSystemRoot)) - throw new ArgumentException($"Cannot find the specified FileSystemRoot: '{Globals.FileSystemOptions.FileSystemRoot}'"); - Logger.Debug("Creating FileCopierHost with FileSystemRoot: " + Globals.FileSystemOptions.FileSystemRoot); IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false); - var fileCopier = new ExtractionFileCopier(Globals.FileCopierOptions, copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot); + var fileCopier = new ExtractionFileCopier(Globals.FileCopierOptions, copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot, fileSystem); _consumer = new FileCopyQueueConsumer(fileCopier); } diff --git a/tests/common/Smi.Common.Tests/MicroserviceTester.cs b/tests/common/Smi.Common.Tests/MicroserviceTester.cs index d681a426d..90e2b748d 100644 --- a/tests/common/Smi.Common.Tests/MicroserviceTester.cs +++ b/tests/common/Smi.Common.Tests/MicroserviceTester.cs @@ -18,7 +18,7 @@ public class MicroserviceTester : IDisposable private readonly List _declaredExchanges = new List(); private readonly List _declaredQueues = new List(); - private readonly ConnectionFactory _factory; + public readonly ConnectionFactory Factory; /// /// When true, will delete any created queues/exchanges when Dispose is called. Can set to false to inspect @@ -41,7 +41,7 @@ public MicroserviceTester(RabbitOptions rabbitOptions, params ConsumerOptions[] _adapter = new RabbitMqAdapter(rabbitOptions.CreateConnectionFactory(), "TestHost"); - _factory = new ConnectionFactory + Factory = new ConnectionFactory { HostName = rabbitOptions.RabbitMqHostName, Port = rabbitOptions.RabbitMqHostPort, @@ -50,7 +50,7 @@ public MicroserviceTester(RabbitOptions rabbitOptions, params ConsumerOptions[] Password = rabbitOptions.RabbitMqPassword }; - using (var con = _factory.CreateConnection()) + using (var con = Factory.CreateConnection()) using (var model = con.CreateModel()) { //get rid of old exchanges @@ -144,7 +144,7 @@ public void CreateExchange(string exchangeName, string queueName = null, bool is string queueNameToUse = queueName ?? exchangeName.Replace("Exchange", "Queue"); - using (var con = _factory.CreateConnection()) + using (var con = Factory.CreateConnection()) using (var model = con.CreateModel()) { //setup a sender channel for each of the consumers you want to test sending messages to @@ -186,7 +186,7 @@ public void Dispose() if (CleanUpAfterTest) { - using (IConnection conn = _factory.CreateConnection()) + using (IConnection conn = Factory.CreateConnection()) using (IModel model = conn.CreateModel()) { _declaredExchanges.ForEach(x => model.ExchangeDelete(x)); diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs index 0b7e6f46a..56afa59bf 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -1,11 +1,24 @@ using Microservices.CohortExtractor.Audit; +using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.ProjectPathResolvers; using Microservices.CohortExtractor.Execution.RequestFulfillers; using Microservices.CohortExtractor.Messaging; using Moq; +using Newtonsoft.Json; using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Framing; +using Smi.Common.Events; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; using Smi.Common.Options; +using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; namespace Microservices.CohortExtractor.Tests.Messaging @@ -15,7 +28,10 @@ public class ExtractionRequestQueueConsumerTest #region Fixture Methods [OneTimeSetUp] - public void OneTimeSetUp() { } + public void OneTimeSetUp() + { + TestLogger.Setup(); + } [OneTimeTearDown] public void OneTimeTearDown() { } @@ -37,32 +53,118 @@ public void TearDown() { } [Test] public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() { - var options = new CohortExtractorOptions(); + GlobalOptions globals = GlobalOptions.Load(); + globals.CohortExtractorOptions.ExtractAnonRoutingKey = "anon"; + globals.CohortExtractorOptions.ExtractIdentRoutingKey = ""; + TestRoutingKeys(globals, false, "anon"); + } + + [Test] + public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() + { + GlobalOptions globals = GlobalOptions.Load(); + globals.CohortExtractorOptions.ExtractAnonRoutingKey = ""; + globals.CohortExtractorOptions.ExtractIdentRoutingKey = "ident"; + TestRoutingKeys(globals, true, "ident"); + } + + private static void TestRoutingKeys(GlobalOptions globals, bool isIdentifiableExtraction, string expectedRoutingKey) + { + // TODO(rkm 2020-08-28) Why do we need this much boilerplate to test a string & a bool? var mockFulfiller = new Mock(MockBehavior.Strict); + mockFulfiller + .Setup(x => x.GetAllMatchingFiles(It.IsAny(), It.IsAny())) + .Returns(() => new List + { + new ExtractImageCollection("foo") + { + { + "bar", new HashSet + { + new QueryToExecuteResult( + "file.dcm", + "study", + "series", + "instance", + false, + "") + } + } + } + }); + var mockAuditor = new Mock(MockBehavior.Strict); + mockAuditor.Setup(x => x.AuditExtractionRequest(It.IsAny())); + mockAuditor.Setup(x => x.AuditExtractFiles(It.IsAny(), It.IsAny())); + var mockPathResolver = new Mock(MockBehavior.Strict); + mockPathResolver + .Setup(x => x.GetOutputPath(It.IsAny(), It.IsAny())) + .Returns("path"); var mockFileMessageProducerModel = new Mock(MockBehavior.Strict); + string fileMessageRoutingKey = null; + mockFileMessageProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsNotNull())) + .Callback((IMessage _, IMessageHeader __, string routingKey) => { fileMessageRoutingKey = routingKey; }) + .Returns(new MessageHeader()); + mockFileMessageProducerModel.Setup(x => x.WaitForConfirms()); + var mockFileInfoMessageProducerModel = new Mock(MockBehavior.Strict); + mockFileInfoMessageProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsNotNull())) + .Returns(new MessageHeader()); + mockFileInfoMessageProducerModel.Setup(x => x.WaitForConfirms()); + + var msg = new ExtractionRequestMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "1234", + ExtractionDirectory = "1234/foo", + IsIdentifiableExtraction = isIdentifiableExtraction, + KeyTag = "foo", + ExtractionIdentifiers = new List { "foo" }, + Modality = null, + }; + var mockDeliverArgs = Mock.Of(MockBehavior.Strict); + mockDeliverArgs.DeliveryTag = 1; + mockDeliverArgs.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(msg)); + mockDeliverArgs.BasicProperties = new BasicProperties { Headers = new Dictionary() }; + var header = new MessageHeader(); + header.Populate(mockDeliverArgs.BasicProperties.Headers); + // Have to convert these to bytes since RabbitMQ normally does that when sending + mockDeliverArgs.BasicProperties.Headers["MessageGuid"] = Encoding.UTF8.GetBytes(header.MessageGuid.ToString()); + mockDeliverArgs.BasicProperties.Headers["ProducerExecutableName"] = Encoding.UTF8.GetBytes(header.ProducerExecutableName); + mockDeliverArgs.BasicProperties.Headers["Parents"] = Encoding.UTF8.GetBytes(string.Join("->", header.Parents)); var consumer = new ExtractionRequestQueueConsumer( - options, + globals.CohortExtractorOptions, mockFulfiller.Object, mockAuditor.Object, mockPathResolver.Object, mockFileMessageProducerModel.Object, mockFileInfoMessageProducerModel.Object); - - // TODO - //consumer.ProcessMessage(); - - Assert.Inconclusive(); - } - [Test] - public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() - { - Assert.Inconclusive(); + var fatalCalled = false; + FatalErrorEventArgs fatalErrorEventArgs = null; + consumer.OnFatal += (sender, args) => + { + fatalCalled = true; + fatalErrorEventArgs = args; + }; + + var mockModel = new Mock(MockBehavior.Strict); + mockModel.Setup(x => x.IsClosed).Returns(false); + mockModel.Setup(x => x.BasicAck(It.IsAny(), It.IsAny())).Verifiable(); + + consumer.SetModel(mockModel.Object); + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal call is race-y + Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); + mockModel.Verify(x => x.BasicAck(It.IsAny(), It.IsAny()), Times.Once); + Assert.AreEqual(expectedRoutingKey, fileMessageRoutingKey); } #endregion diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs index 49fceb8c2..4cf0a77c2 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -1,4 +1,17 @@ -using NUnit.Framework; +using Microservices.FileCopier.Execution; +using Newtonsoft.Json; +using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using Smi.Common.Messages.Extraction; +using Smi.Common.Options; +using Smi.Common.Tests; +using System; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Text; +using System.Threading; + namespace Microservices.FileCopier.Tests.Execution { @@ -7,7 +20,10 @@ public class FileCopierHostTest #region Fixture Methods [OneTimeSetUp] - public void OneTimeSetUp() { } + public void OneTimeSetUp() + { + TestLogger.Setup(); + } [OneTimeTearDown] public void OneTimeTearDown() { } @@ -29,7 +45,57 @@ public void TearDown() { } [Test] public void Test_FileCopierHost_HappyPath() { - Assert.Inconclusive(); + GlobalOptions globals = GlobalOptions.Load(); + globals.FileSystemOptions.FileSystemRoot = "root"; + globals.FileSystemOptions.ExtractRoot = "exroot"; + + using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions); + + string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue"); + tester.CreateExchange( + globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName, + outputQueueName, + false, + globals.FileCopierOptions.NoVerifyRoutingKey); + + var mockFileSystem = new MockFileSystem(); + mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot); + mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot); + mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject); + + var host = new FileCopierHost(globals, mockFileSystem, false); + host.Start(); + + var message = new ExtractFileMessage + { + ExtractionJobIdentifier = Guid.NewGuid(), + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "1234", + ExtractionDirectory = "1234/foo", + DicomFilePath = "file.dcm", + IsIdentifiableExtraction = true, + OutputPath = "output.dcm", + }; + tester.SendMessage(globals.FileCopierOptions, message); + + using IConnection conn = tester.Factory.CreateConnection(); + using IModel model = conn.CreateModel(); + + var timeout = 5; + while (--timeout >= 0 && model.MessageCount(outputQueueName) == 0) + { + Thread.Sleep(1000); + } + Assert.True(timeout >= 0); + + host.Stop("test finished"); + + var consumer = new EventingBasicConsumer(model); + ExtractedFileStatusMessage statusMessage = null; + consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(ea.Body.ToArray())); + model.BasicConsume(outputQueueName, true, "", consumer); + Thread.Sleep(500); // TODO Race-y + Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status); } #endregion From cc395be03db675faea678af53ca80d7ecc5526e8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 14:33:08 +0100 Subject: [PATCH 067/138] Implement FileCopyQueueConsumer tests --- .../Messaging/FileCopyQueueConsumerTest.cs | 151 ++++++++++++++++-- 1 file changed, 136 insertions(+), 15 deletions(-) diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs index a15e77c6a..7bee3acca 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -1,4 +1,20 @@ -using NUnit.Framework; +using Microservices.FileCopier.Execution; +using Microservices.FileCopier.Messaging; +using Moq; +using Newtonsoft.Json; +using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Framing; +using Smi.Common.Events; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; + namespace Microservices.FileCopier.Tests.Messaging { @@ -6,8 +22,15 @@ public class FileCopyQueueConsumerTest { #region Fixture Methods + private ExtractFileMessage _message; + private Mock _mockModel; + private Mock _mockFileCopier; + [OneTimeSetUp] - public void OneTimeSetUp() { } + public void OneTimeSetUp() + { + TestLogger.Setup(); + } [OneTimeTearDown] public void OneTimeTearDown() { } @@ -17,11 +40,45 @@ public void OneTimeTearDown() { } #region Test Methods [SetUp] - public void SetUp() { } + public void SetUp() + { + _message = new ExtractFileMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "1234", + ExtractionDirectory = "foo", + DicomFilePath = "foo.dcm", + IsIdentifiableExtraction = true, + OutputPath = "bar", + }; + _mockModel = new Mock(MockBehavior.Strict); + _mockModel.Setup(x => x.IsClosed).Returns(false); + _mockModel.Setup(x => x.BasicAck(It.IsAny(), It.IsAny())); + _mockModel.Setup(x => x.BasicNack(It.IsAny(), It.IsAny(), It.IsAny())); + + _mockFileCopier = new Mock(MockBehavior.Strict); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())); + } [TearDown] public void TearDown() { } + private static BasicDeliverEventArgs GetMockDeliverArgs(ExtractFileMessage message) + { + var mockDeliverArgs = Mock.Of(MockBehavior.Strict); + mockDeliverArgs.DeliveryTag = 1; + mockDeliverArgs.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message)); + mockDeliverArgs.BasicProperties = new BasicProperties { Headers = new Dictionary() }; + var header = new MessageHeader(); + header.Populate(mockDeliverArgs.BasicProperties.Headers); + // Have to convert these to bytes since RabbitMQ normally does that when sending + mockDeliverArgs.BasicProperties.Headers["MessageGuid"] = Encoding.UTF8.GetBytes(header.MessageGuid.ToString()); + mockDeliverArgs.BasicProperties.Headers["ProducerExecutableName"] = Encoding.UTF8.GetBytes(header.ProducerExecutableName); + mockDeliverArgs.BasicProperties.Headers["Parents"] = Encoding.UTF8.GetBytes(string.Join("->", header.Parents)); + return mockDeliverArgs; + } + #endregion #region Tests @@ -29,33 +86,97 @@ public void TearDown() { } [Test] public void Test_FileCopyQueueConsumer_ValidMessage_IsAcked() { - // There's a ridiculous amount of boilerplate required to test this at the moment... - //var mockFileCopier = new Mock(MockBehavior.Strict); - //var consumer = new FileCopyQueueConsumer(mockFileCopier.Object); - //consumer.ProcessMessage(); - - // TODO(rkm 2020-08-25) Test Ack / not Nack - Assert.Inconclusive(); + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + FatalErrorEventArgs fatalErrorEventArgs = null; + consumer.OnFatal += (sender, args) => + { + fatalCalled = true; + fatalErrorEventArgs = args; + }; + + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal is race-y + Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); + Assert.AreEqual(1, consumer.AckCount); + Assert.AreEqual(0, consumer.NackCount); } [Test] public void Test_FileCopyQueueConsumer_ApplicationException_IsNacked() { - // TODO(rkm 2020-08-25) Test Nack / not Ack - Assert.Inconclusive(); + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + FatalErrorEventArgs fatalErrorEventArgs = null; + consumer.OnFatal += (sender, args) => + { + fatalCalled = true; + fatalErrorEventArgs = args; + }; + + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal is race-y + Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); + Assert.AreEqual(0, consumer.AckCount); + Assert.AreEqual(1, consumer.NackCount); } [Test] public void Test_FileCopyQueueConsumer_UnknownException_CallsFatalCallback() { - // TODO(rkm 2020-08-25) Test not ack / not nack / Fatal called - Assert.Inconclusive(); + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + consumer.OnFatal += (sender, _) => fatalCalled = true; + + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal is race-y + Assert.True(fatalCalled, "Expected Fatal to be called"); + Assert.AreEqual(0, consumer.AckCount); + Assert.AreEqual(0, consumer.NackCount); } [Test] public void Test_FileCopyQueueConsumer_AnonExtraction_ThrowsException() { - Assert.Inconclusive(); + _message.IsIdentifiableExtraction = false; + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + consumer.OnFatal += (sender, _) => fatalCalled = true; + + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal is race-y + Assert.True(fatalCalled, "Expected Fatal to be called"); + Assert.AreEqual(0, consumer.AckCount); + Assert.AreEqual(0, consumer.NackCount); } #endregion From b953ca49bfde9d1bc50c3009d89c301082c282ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2020 14:53:27 +0000 Subject: [PATCH 068/138] Bump HIC.RDMP.Dicom from 2.1.8 to 2.1.9 (#374) --- PACKAGES.md | 2 +- .../Microservices.DicomRelationalMapper.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index eebaf3a25..015f9e442 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -14,7 +14,7 @@ | fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.8](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.9](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.9) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index 855052aea..33d39261f 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -18,7 +18,7 @@ - + From 633f16586f8e0025627770800accf02e89eace4a Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 17:30:26 +0100 Subject: [PATCH 069/138] Handle old format in MongoDB extraction documents --- .../ObjectModel/MongoExpectedFilesDoc.cs | 8 +-- .../MongoDB/ObjectModel/MongoFileStatusDoc.cs | 32 ++++++++-- .../MongoCompletedExtractJobDocTest.cs | 43 ++++++++++++-- .../ObjectModel/MongoExtractJobDocTest.cs | 12 ---- .../ObjectModel/MongoFileStatusDocTest.cs | 58 +++++++++++++++++++ 5 files changed, 128 insertions(+), 25 deletions(-) diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs index cee799505..b9b4230d7 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; -using System.Linq; using JetBrains.Annotations; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; +using System; +using System.Collections.Generic; +using System.Linq; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -14,7 +14,7 @@ namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.Objec /// /// MongoDB document model representing a set of files which are expected to be extracted /// - [BsonIgnoreExtraElements] + [BsonIgnoreExtraElements] // NOTE(rkm 2020-08-28) Required for classes which don't contain a field marked with BsonId public class MongoExpectedFilesDoc : IEquatable { [BsonElement("header")] diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs index fc63c2cc0..19d0299ec 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs @@ -3,14 +3,14 @@ using MongoDB.Bson.Serialization.Attributes; using Smi.Common.Messages.Extraction; using System; +using System.Collections.Generic; +using System.ComponentModel; + namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel { - /// - /// - /// - [BsonIgnoreExtraElements] - public class MongoFileStatusDoc + [BsonIgnoreExtraElements] // NOTE(rkm 2020-08-28) Required for classes which don't contain a field marked with BsonId + public class MongoFileStatusDoc : ISupportInitialize { [BsonElement("header")] [NotNull] @@ -41,6 +41,14 @@ public class MongoFileStatusDoc [CanBeNull] public string StatusMessage { get; set; } + /// + /// Used only to handle old-format documents when deserializing + /// + [BsonExtraElements] + [UsedImplicitly] + public IDictionary ExtraElements { get; set; } + + public MongoFileStatusDoc( [NotNull] MongoExtractionMessageHeaderDoc header, [NotNull] string dicomFilePath, @@ -61,6 +69,20 @@ public MongoFileStatusDoc( throw new ArgumentNullException(nameof(statusMessage)); } + // ^ISupportInitialize + public void BeginInit() { } + + // ^ISupportInitialize + public void EndInit() + { + if (!ExtraElements.ContainsKey("anonymisedFileName")) + return; + + OutputFileName = (string)ExtraElements["anonymisedFileName"]; + DicomFilePath = ""; + ExtractedFileStatus = OutputFileName == null ? ExtractedFileStatus.ErrorWontRetry : ExtractedFileStatus.Anonymised; + } + #region Equality Methods protected bool Equals(MongoFileStatusDoc other) diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs index 371635459..7f75cb7a0 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs @@ -1,11 +1,13 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel { @@ -48,13 +50,46 @@ public void TearDown() { } #region Tests + [Test] + public void Test_MongoCompletedExtractJobDoc_ParseOldFormat() + { + Console.WriteLine(Guid.NewGuid()); + const string jsonDoc = @" +{ + '_id' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'header' : { + 'extractionJobIdentifier' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'messageGuid' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'producerExecutableName' : 'ExtractorCL', + 'producerProcessID' : 1234, + 'originalPublishTimestamp' : ISODate('2020-08-28T12:00:00Z'), + 'parents' : '', + 'receivedAt' : ISODate('2020-08-28T12:00:00Z') + }, + 'projectNumber' : '1234s', + 'jobStatus' : 'Completed', + 'extractionDirectory' : 'foo/bar', + 'jobSubmittedAt' : ISODate('2020-08-28T12:00:00Z'), + 'keyTag' : 'SeriesInstanceUID', + 'keyCount' : 123, + 'extractionModality' : null, + 'failedJobInfo' : null, + 'completedAt' : ISODate('2020-08-28T12:00:00Z'), +}"; + + var mongoExtractJobDoc = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + // NOTE(rkm 2020-08-28) This works by chance since the missing bool will default to false, so we don't require MongoCompletedExtractJobDoc to implement ISupportInitialize + Assert.False(mongoExtractJobDoc.IsIdentifiableExtraction); + } + [Test] public void TestMongoCompletedExtractJobDoc_SettersAvailable() { foreach (PropertyInfo p in typeof(MongoCompletedExtractJobDoc).GetProperties()) Assert.True(p.CanWrite, $"Property '{p.Name}' is not writeable"); } - + [Test] public void TestMongoCompletedExtractJobDoc_Constructor_ExtractJobStatus() { diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs index 69de742f3..806432297 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs @@ -1,7 +1,5 @@ using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; -using MongoDB.Bson; -using MongoDB.Bson.Serialization; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; @@ -151,16 +149,6 @@ public void TestMongoExtractJobDoc_GetHashCode() Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); } - [Test] - public void TestMongoExtractJobDoc_IsIdentifiableExtraction_MissingValueOk() - { - // TODO(rkm 2020-08-27) This works by chance since the missing boolean value defaults to false anyway. Need to think of a better way of handling this kind of backwards compatibility - var jsonDoc = "{ \"_id\" : \"0fbd4893-c116-4f16-88c8-f7084531d87c\", \"header\" : { \"extractionJobIdentifier\" : \"0fbd4893-c116-4f16-88c8-f7084531d87c\", \"messageGuid\" : \"23475d89-6e2c-431c-bc5d-7b7c25ffb6a0\", \"producerExecutableName\" : \"testhost\", \"producerProcessID\" : 14372, \"originalPublishTimestamp\" : { \"$date\" : 1598528178000 }, \"parents\" : \"30603fb0-3bec-43de-8ba8-db55a029c664\", \"receivedAt\" : { \"$date\" : 1598531778957 } }, \"projectNumber\" : \"1234\", \"jobStatus\" : \"WaitingForCollectionInfo\", \"extractionDirectory\" : \"test/directory\", \"jobSubmittedAt\" : { \"$date\" : 1598531778957 }, \"keyTag\" : \"KeyTag\", \"keyCount\" : 123, \"extractionModality\" : \"MR\",\"failedJobInfo\" : null }"; - BsonDocument bsonDoc = BsonDocument.Parse(jsonDoc); - var mongoExtractJobDoc = BsonSerializer.Deserialize(bsonDoc); - Assert.False(mongoExtractJobDoc.IsIdentifiableExtraction); - } - [Test] public void TestMongoFailedJobInfoDoc_SettersAvailable() { diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs index 311692e8d..6b3d53337 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs @@ -1,4 +1,6 @@ using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; @@ -64,6 +66,62 @@ public void Test_MongoFileStatusDoc_IsIdentifiable_StatusMessage() null)); } + [Test] + public void Test_MongoFileStatusDoc_ParseOldFormat_VerificationMessage() + { + // NOTE(rkm 2020-08-28) Format as of release v1.11.1 + const string jsonDoc = @" +{ + '_id' : ObjectId('5f490ef8473b9739448cbe4c'), + 'header': { + 'extractionJobIdentifier':'f9586843-8dbb-46a6-b36d-4646fdfddede', + 'messageGuid': '21a63ac3-c6f0-4fb9-973c-c97490920246', + 'producerExecutableName':'IsIdentifiable', + 'producerProcessID': 1234, + 'originalPublishTimestamp': ISODate('2020-08-28T12:00:00.000Z'), + 'parents': 'cd6430dc-952e-420e-808c-7910e61e9278->a9e16701-ef8b-482c-8b1b-023f6f40fdde->cc84ebbc-ebd0-40d0-a7da-8a2c5004b8bc', + 'receivedAt': ISODate('2020-08-28T12:00:00.000Z') + }, + 'anonymisedFileName' : 'anon.dcm', + 'wasAnonymised' : true, + 'isIdentifiable' : false, + 'statusMessage' : '[]' +}"; + var parsed = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + Assert.AreEqual("anon.dcm", parsed.OutputFileName); + Assert.AreEqual("", parsed.DicomFilePath); + Assert.AreEqual(ExtractedFileStatus.Anonymised, parsed.ExtractedFileStatus); + } + + [Test] + public void Test_MongoFileStatusDoc_ParseOldFormat_AnonFailedMessage() + { + // NOTE(rkm 2020-08-28) Format as of release v1.11.1 + const string jsonDoc = @" +{ + '_id' : ObjectId('5f490ef8473b9739448cbe4c'), + 'header': { + 'extractionJobIdentifier':'f9586843-8dbb-46a6-b36d-4646fdfddede', + 'messageGuid': '21a63ac3-c6f0-4fb9-973c-c97490920246', + 'producerExecutableName':'CTPAnonymiser', + 'producerProcessID': 1234, + 'originalPublishTimestamp': ISODate('2020-08-28T12:00:00.000Z'), + 'parents': 'cd6430dc-952e-420e-808c-7910e61e9278->a9e16701-ef8b-482c-8b1b-023f6f40fdde', + 'receivedAt': ISODate('2020-08-28T12:00:00.000Z') + }, + 'anonymisedFileName' : null, + 'wasAnonymised' : false, + 'isIdentifiable' : false, + 'statusMessage' : 'failed to anonymise' +}"; + var parsed = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + Assert.AreEqual(null, parsed.OutputFileName); + Assert.AreEqual("", parsed.DicomFilePath); + Assert.AreEqual(ExtractedFileStatus.ErrorWontRetry, parsed.ExtractedFileStatus); + } + [Test] public void TestMongoFileStatusDoc_SettersAvailable() { From dc6ecc21cb96afc3bd87e8195b26fd9523cc05a2 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 17:34:36 +0100 Subject: [PATCH 070/138] Update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8b06c8b8..aab4f07cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Extraction report: Group PixelData separately and sort by length - Fix the extraction output directory to be `/extractions/` - Add identifiable extraction support + - New service "FileCopier" which sits in place of CTP for identifiable extractions and copies source files to their output dirs + - Changes to MongoDB extraction schema, but backwards compatibility has been tested - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated - - [breaking] Changes to MongoDB extraction schema. Existing databases need to be updated ## [1.11.1] - 2020-08-12 From 7c9ea5210c9c820bc39c969305d27157edf929ae Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 18:22:02 +0100 Subject: [PATCH 071/138] Add IsNoFilterExtraction to extraction metadata --- .../java/org/smi/extractorcl/Program.java | 9 ++++++ .../execution/ExtractorClHost.java | 4 ++- .../fileUtils/ExtractMessagesCsvHandler.java | 14 +++++--- .../Messages/Extraction/ExtractMessage.cs | 32 +++++++++++++++---- .../Messages/Extraction/IExtractMessage.cs | 5 +++ .../smi/common/messages/ExtractMessage.java | 8 ++++- 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java index 74dd579c8..91fe057ca 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java @@ -96,6 +96,15 @@ private static CommandLine ParseOptions(String[] args) throws ParseException { .argName("identifiable extraction") .desc("This is an identifiable extraction") .longOpt("identifiable-extraction") + .build()); + + options.addOption( + Option + .builder("f") + .type(boolean.class) + .argName("no-filters extraction") + .desc("Extraction with no reject filters. True by default if --identifiable-extraction specified") + .longOpt("no-filters-extraction") .build()); try { diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index 290fd3a9e..1c722871f 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -68,10 +68,12 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU final String extractionDir = projectID + "/extractions/" + extractionName; final boolean isIdentifiableExtraction = commandLineOptions.hasOption("i"); + final boolean isNoFilterExtraction = commandLineOptions.hasOption("f") || isIdentifiableExtraction; _logger.debug("projectID: " + projectID); _logger.debug("extractionDirectory: " + extractionDir); _logger.debug("isIdentifiableExtraction: " + isIdentifiableExtraction); + _logger.debug("isNoFilterExtraction: " + isNoFilterExtraction); Path fullExtractionDirectory = Paths.get(extractionRoot.getAbsolutePath().toString(), extractionDir); @@ -88,7 +90,7 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionModality = commandLineOptions.getOptionValue("modality", null); - _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, isIdentifiableExtraction, + _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, isIdentifiableExtraction, isNoFilterExtraction, rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestProducerOptions), rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestInfoProducerOptions)); } diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java index 7c993b932..a70b5a968 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java @@ -30,6 +30,7 @@ public class ExtractMessagesCsvHandler implements CsvHandler { private String _extractionDir; private String _extractionModality; private boolean _isIdentifiableExtraction; + private boolean _isNoFilterExtraction; private ExtractionKey _extractionKey; private static final Pattern _chiPattern = Pattern.compile("^\\d{10}$"); private static final Pattern _eupiPattern = Pattern.compile("^([A-Z]|[0-9]){32}$"); @@ -52,7 +53,7 @@ public class ExtractMessagesCsvHandler implements CsvHandler { * ExtractRequestInfo messages */ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String extractionDir, - String extractionModality, boolean isIdentifiableExtraction, IProducerModel extractRequestMessageProducerModel, + String extractionModality, boolean isIdentifiableExtraction, boolean isNoFilterExtraction, IProducerModel extractRequestMessageProducerModel, IProducerModel extractRequestInfoMessageProducerModel) { _extractionJobID = extractionJobID; @@ -62,6 +63,7 @@ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String _extractRequestInfoMessageProducerModel = extractRequestInfoMessageProducerModel; _extractionModality = extractionModality; _isIdentifiableExtraction = isIdentifiableExtraction; + _isNoFilterExtraction = isNoFilterExtraction; // TODO(rkm 2020-01-30) Properly handle parsing of the supported modalities if (_extractionModality != null && (!_extractionModality.equals("CT") && !_extractionModality.equals("MR"))) { @@ -149,7 +151,8 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erm.ExtractionDirectory = _extractionDir; erm.JobSubmittedAt = now; erm.KeyTag = _extractionKey.toString(); - erm.IsIdentifiableExtraction = _isIdentifiableExtraction; + erm.IsIdentifiableExtraction = _isIdentifiableExtraction; + erm.IsNoFilterExtraction = _isNoFilterExtraction; if (_extractionKey == ExtractionKey.StudyInstanceUID) erm.ExtractionModality = _extractionModality; @@ -161,7 +164,9 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erim.JobSubmittedAt = now; erim.KeyValueCount = _identifierSet.size(); erim.KeyTag = _extractionKey.toString(); - erim.IsIdentifiableExtraction = _isIdentifiableExtraction; + erim.IsIdentifiableExtraction = _isIdentifiableExtraction; + erim.IsNoFilterExtraction = _isNoFilterExtraction; + if (_extractionKey == ExtractionKey.StudyInstanceUID) erim.ExtractionModality = _extractionModality; @@ -170,7 +175,8 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I sb.append(" ProjectNumber: " + _projectID + System.lineSeparator()); sb.append(" ExtractionDirectory: " + _extractionDir + System.lineSeparator()); sb.append(" ExtractionKey: " + _extractionKey + System.lineSeparator()); - sb.append(" IsIdentifiableExtraction: " + _isIdentifiableExtraction + System.lineSeparator()); + sb.append(" IsIdentifiableExtraction: " + _isIdentifiableExtraction + System.lineSeparator()); + sb.append(" IsNoFilterExtraction: " + _isNoFilterExtraction + System.lineSeparator()); if (_extractionKey == ExtractionKey.StudyInstanceUID) sb.append(" ExtractionModality: " + _extractionModality + System.lineSeparator()); sb.append(" KeyValueCount: " + _identifierSet.size() + System.lineSeparator()); diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs index 89512aac0..675449a6f 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs @@ -1,7 +1,9 @@ +using JetBrains.Annotations; using Newtonsoft.Json; using System; + namespace Smi.Common.Messages.Extraction { /// @@ -24,11 +26,20 @@ public abstract class ExtractMessage : IExtractMessage, IEquatable @@ -53,6 +66,7 @@ public override string ToString() => $"ExtractionDirectory={ExtractionDirectory}, " + $"JobSubmittedAt={JobSubmittedAt}, " + $"IsIdentifiableExtraction={IsIdentifiableExtraction}, " + + $"IsNoFilterExtraction={IsNoFilterExtraction}, " + ""; #region Equality Members @@ -62,11 +76,14 @@ public bool Equals(ExtractMessage other) if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return - ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) && - string.Equals(ProjectNumber, other.ProjectNumber) && - string.Equals(ExtractionDirectory, other.ExtractionDirectory) && - JobSubmittedAt.Equals(other.JobSubmittedAt); + return true + && ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) + && string.Equals(ProjectNumber, other.ProjectNumber) + && string.Equals(ExtractionDirectory, other.ExtractionDirectory) + && JobSubmittedAt.Equals(other.JobSubmittedAt) + && IsIdentifiableExtraction == other.IsIdentifiableExtraction + && IsNoFilterExtraction == other.IsNoFilterExtraction + && true; } public override bool Equals(object obj) @@ -86,6 +103,7 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode(); hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); return hashCode; } } diff --git a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs index 6ee3b5379..f6937b31c 100644 --- a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs @@ -35,5 +35,10 @@ public interface IExtractMessage : IMessage /// True if this is an identifiable extraction (i.e. files should not be anonymised) /// bool IsIdentifiableExtraction { get; } + + /// + /// True if this is a "no filters" (i.e. no file rejection filters should be applied) + /// + bool IsNoFilterExtraction { get; } } } diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java index e73e97938..6f7581241 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java @@ -36,8 +36,14 @@ public abstract class ExtractMessage implements IMessage { * True if this is an identifiable extraction (i.e. files should not be anonymised) */ @FieldRequired - public boolean IsIdentifiableExtraction; + public boolean IsIdentifiableExtraction; + /** + * True if this is a "no filters" (i.e. no file rejection filters should be applied) + */ + @FieldRequired + public boolean IsNoFilterExtraction; + protected ExtractMessage() { } } From e127637cf7b9d50d2ab20f9a536cf37f178e5626 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 18:22:47 +0100 Subject: [PATCH 072/138] Skip rejectors in FromCataloguesExtractionRequestFulfiller when required --- ...romCataloguesExtractionRequestFulfiller.cs | 22 ++--- ...taloguesExtractionRequestFulfillerTests.cs | 80 +++++++++++++++---- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs index 9b80a3bed..cf7a83be4 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs @@ -1,8 +1,8 @@ using Microservices.CohortExtractor.Audit; -using Smi.Common.Messages.Extraction; using NLog; using Rdmp.Core.Curation.Data; +using Smi.Common.Messages.Extraction; using System; using System.Collections.Generic; using System.Linq; @@ -41,7 +41,7 @@ protected QueryToExecuteColumnSet[] FilterCatalogues(ICatalogue[] cataloguesToUs public IEnumerable GetAllMatchingFiles(ExtractionRequestMessage message, IAuditExtractions auditor) { var queries = new List(); - + foreach (var c in GetCataloguesFor(message)) { var query = GetQueryToExecute(c, message); @@ -53,12 +53,12 @@ public IEnumerable GetAllMatchingFiles(ExtractionRequest } } - Logger.Debug("Found " + queries.Count + " Catalogues which support extracting based on '" + message.KeyTag + "'"); + Logger.Debug($"Found {queries.Count} Catalogues which support extracting based on '{message.KeyTag}'"); if (queries.Count == 0) throw new Exception($"Couldn't find any compatible Catalogues to run extraction queries against for query {message}"); - + List rejectorsToUse = message.IsNoFilterExtraction ? new List() : Rejectors; foreach (string valueToLookup in message.ExtractionIdentifiers) { @@ -66,10 +66,10 @@ public IEnumerable GetAllMatchingFiles(ExtractionRequest foreach (QueryToExecute query in queries) { - foreach (var result in query.Execute(valueToLookup,Rejectors)) + foreach (QueryToExecuteResult result in query.Execute(valueToLookup, rejectorsToUse)) { - if(!results.ContainsKey(result.SeriesTagValue)) - results.Add(result.SeriesTagValue,new HashSet()); + if (!results.ContainsKey(result.SeriesTagValue)) + results.Add(result.SeriesTagValue, new HashSet()); results[result.SeriesTagValue].Add(result); } @@ -90,7 +90,7 @@ protected virtual QueryToExecute GetQueryToExecute(QueryToExecuteColumnSet colum { if (!string.IsNullOrWhiteSpace(message.Modality)) { - if(ModalityRoutingRegex == null) + if (ModalityRoutingRegex == null) throw new NotSupportedException("Filtering on Modality requires setting a ModalityRoutingRegex"); var anyModality = message.Modality.Split(',', StringSplitOptions.RemoveEmptyEntries); @@ -104,12 +104,12 @@ protected virtual QueryToExecute GetQueryToExecute(QueryToExecuteColumnSet colum } else { - Logger.Log(LogLevel.Warn,nameof(ModalityRoutingRegex) + " did not match Catalogue name " + columnSet.Catalogue.Name); + Logger.Log(LogLevel.Warn, nameof(ModalityRoutingRegex) + " did not match Catalogue name " + columnSet.Catalogue.Name); } } - - + + return new QueryToExecute(columnSet, message.KeyTag); } diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs index 587a6987b..6e4815e62 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs @@ -1,6 +1,9 @@  using FAnsi; +using FAnsi.Discovery; +using FAnsi.Extensions; using Microservices.CohortExtractor.Audit; +using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.RequestFulfillers; using NUnit.Framework; using Rdmp.Core.Curation.Data; @@ -9,7 +12,6 @@ using System.Collections.Generic; using System.Data; using System.Linq; -using FAnsi.Extensions; using Tests.Common; namespace Microservices.CohortExtractor.Tests @@ -19,8 +21,13 @@ namespace Microservices.CohortExtractor.Tests /// (described in a ) and fetch matching image urls out of the database (creating /// ExtractImageCollection results). /// - class FromCataloguesExtractionRequestFulfillerTests : DatabaseTests + public class FromCataloguesExtractionRequestFulfillerTests : DatabaseTests { + [SetUp] + public void SetUp() + { + TestLogger.Setup(); + } [TestCase(DatabaseType.MicrosoftSQLServer), RequiresRelationalDb(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql), RequiresRelationalDb(DatabaseType.MySql)] @@ -30,7 +37,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching_SeriesInstan var dt = new DataTable(); dt.Columns.Add("SeriesInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); dt.Rows.Add("123", true, "/images/1.dcm"); @@ -56,7 +63,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching_SeriesInstan Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/2.dcm"))); } - + [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInstanceUIDOnly(DatabaseType databaseType) @@ -65,7 +72,7 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInsta var dt = new DataTable(); dt.Columns.Add("SeriesInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); dt.Rows.Add("123", true, "/images/1.dcm"); @@ -95,7 +102,7 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInsta Assert.AreEqual(1, matching[0].Accepted.Count); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); } - + [TestCase(DatabaseType.MicrosoftSQLServer), RequiresRelationalDb(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql), RequiresRelationalDb(DatabaseType.MySql)] @@ -110,10 +117,10 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching(DatabaseType dt.Columns.Add("Extractable"); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); - dt.Rows.Add("1.1","123.1","1.1", true, "/images/1.dcm"); - dt.Rows.Add("1.1","123.1","2.1", false, "/images/2.dcm"); - dt.Rows.Add("1.1","1234.1","3.1", false, "/images/3.dcm"); - dt.Rows.Add("1.1","1234.1","4.1", true, "/images/4.dcm"); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.Rows.Add("1.1", "123.1", "2.1", false, "/images/2.dcm"); + dt.Rows.Add("1.1", "1234.1", "3.1", false, "/images/3.dcm"); + dt.Rows.Add("1.1", "1234.1", "4.1", true, "/images/4.dcm"); dt.SetDoNotReType(true); @@ -133,6 +140,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching(DatabaseType Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/2.dcm"))); } + [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseType databaseType) @@ -143,13 +151,13 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseTyp dt.Columns.Add("StudyInstanceUID"); dt.Columns.Add("SeriesInstanceUID"); dt.Columns.Add("SOPInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); - dt.Rows.Add("1.1","123.1","1.1", true, "/images/1.dcm"); - dt.Rows.Add("1.1","123.1","2.1", false, "/images/2.dcm"); - dt.Rows.Add("1.1","1234.1","3.1", false, "/images/3.dcm"); - dt.Rows.Add("1.1","1234.1","4.1", true, "/images/4.dcm"); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.Rows.Add("1.1", "123.1", "2.1", false, "/images/2.dcm"); + dt.Rows.Add("1.1", "1234.1", "3.1", false, "/images/3.dcm"); + dt.Rows.Add("1.1", "1234.1", "4.1", true, "/images/4.dcm"); dt.SetDoNotReType(true); @@ -173,5 +181,47 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseTyp Assert.AreEqual(1, matching[0].Accepted.Count); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); } + + [TestCase(DatabaseType.MicrosoftSQLServer, true)] + [TestCase(DatabaseType.MicrosoftSQLServer, false)] + public void Test_FromCataloguesExtractionRequestFulfiller_NoFilterExtraction(DatabaseType databaseType, bool isNoFiltersExtraction) + { + DiscoveredDatabase db = GetCleanedServer(databaseType); + + var dt = new DataTable(); + dt.Columns.Add("StudyInstanceUID"); + dt.Columns.Add("SeriesInstanceUID"); + dt.Columns.Add("SOPInstanceUID"); + dt.Columns.Add("Extractable", typeof(bool)); + dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.SetDoNotReType(true); + + DiscoveredTable tbl = db.CreateTable("FromCataloguesExtractionRequestFulfillerTests", dt); + Catalogue catalogue = Import(tbl); + + ExtractionInformation ei = catalogue.GetAllExtractionInformation(ExtractionCategory.Any).First(); + var filter = new ExtractionFilter(CatalogueRepository, "Extractable only", ei) + { + IsMandatory = true, + WhereSQL = "Extractable = 1" + }; + filter.SaveToDatabase(); + var fulfiller = new FromCataloguesExtractionRequestFulfiller(new[] { catalogue }); + fulfiller.Rejectors.Add(new RejectAll()); + + var message = new ExtractionRequestMessage + { + KeyTag = "SeriesInstanceUID", + ExtractionIdentifiers = new List(new [] { "123.1" }), + IsNoFilterExtraction = isNoFiltersExtraction, + }; + + ExtractImageCollection[] matching = fulfiller.GetAllMatchingFiles(message, new NullAuditExtractions()).ToArray(); + + int expected = isNoFiltersExtraction ? 1 : 0; + Assert.AreEqual(1, matching.Length); + Assert.AreEqual(expected, matching[0].Accepted.Count); + } } } From a9a6f666367bff075ff3aede1ad70759cd45fce4 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 18:48:54 +0100 Subject: [PATCH 073/138] Add IsNoFilterExtraction to CohortPackager store & report --- .../ExtractJobStorage/ExtractJobInfo.cs | 10 +- .../MongoDB/MongoExtractJobInfoExtensions.cs | 4 +- .../MongoCompletedExtractJobDoc.cs | 1 - .../MongoDB/ObjectModel/MongoExtractJobDoc.cs | 15 ++- .../Reporting/JobReporterBase.cs | 2 + .../ExtractJobStorage/ExtractJobInfoTest.cs | 20 ++-- .../MongoExtractJobInfoExtensionsTest.cs | 9 +- .../MongoDB/MongoExtractJobStoreTest.cs | 14 ++- .../MongoCompletedExtractJobDocTest.cs | 4 +- .../ObjectModel/MongoExtractJobDocTest.cs | 17 ++- .../JobProcessing/ExtractJobWatcherTest.cs | 7 +- .../Reporting/JobReporterBaseTest.cs | 102 ++++++++++++++++-- 12 files changed, 169 insertions(+), 36 deletions(-) diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs index ae10d0919..e2718f1c2 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs @@ -55,6 +55,8 @@ public class ExtractJobInfo : IEquatable public bool IsIdentifiableExtraction { get; } + public bool IsNoFilterExtraction { get; } + public ExtractJobInfo( Guid extractionJobIdentifier, @@ -65,7 +67,9 @@ public ExtractJobInfo( uint keyValueCount, [CanBeNull] string extractionModality, ExtractJobStatus jobStatus, - bool isIdentifiableExtraction) + bool isIdentifiableExtraction, + bool isNoFilterExtraction + ) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentNullException(nameof(extractionJobIdentifier)); JobSubmittedAt = (jobSubmittedAt != default(DateTime)) ? jobSubmittedAt : throw new ArgumentNullException(nameof(jobSubmittedAt)); @@ -77,6 +81,7 @@ public ExtractJobInfo( ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); JobStatus = (jobStatus != default(ExtractJobStatus)) ? jobStatus : throw new ArgumentException(nameof(jobStatus)); IsIdentifiableExtraction = isIdentifiableExtraction; + IsNoFilterExtraction = isNoFilterExtraction; } public override string ToString() @@ -89,6 +94,7 @@ public override string ToString() sb.AppendLine("KeyCount: " + KeyValueCount); sb.AppendLine("ExtractionModality: " + ExtractionModality); sb.AppendLine("IsIdentifiableExtraction: " + IsIdentifiableExtraction); + sb.AppendLine("IsNoFilterExtraction: " + IsNoFilterExtraction); return sb.ToString(); } @@ -106,6 +112,7 @@ public bool Equals(ExtractJobInfo other) KeyValueCount == other.KeyValueCount && ExtractionModality == other.ExtractionModality && IsIdentifiableExtraction == other.IsIdentifiableExtraction && + IsNoFilterExtraction == other.IsNoFilterExtraction && JobStatus == other.JobStatus; } @@ -134,6 +141,7 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (int) JobStatus; hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); return hashCode; } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs index a93c8165d..0ae3a8bde 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs @@ -14,6 +14,8 @@ public static ExtractJobInfo ToExtractJobInfo(this MongoExtractJobDoc mongoExtra mongoExtractJobDoc.KeyCount, mongoExtractJobDoc.ExtractionModality, mongoExtractJobDoc.JobStatus, - mongoExtractJobDoc.IsIdentifiableExtraction); + mongoExtractJobDoc.IsIdentifiableExtraction, + mongoExtractJobDoc.IsNoFilterExtraction + ); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs index 36a099286..3530ed5fd 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs @@ -8,7 +8,6 @@ namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.Objec { public class MongoCompletedExtractJobDoc : MongoExtractJobDoc, IEquatable { - //TODO(rkm 2020-03-09) Check this is generated by the Bson mapping (derived field) [BsonElement("completedAt")] public DateTime CompletedAt { get; set; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs index c0aa79a09..1b0e7c4a6 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs @@ -1,10 +1,10 @@ -using System; using JetBrains.Annotations; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; +using System; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -48,6 +48,9 @@ public class MongoExtractJobDoc [BsonElement("isIdentifiableExtraction")] public bool IsIdentifiableExtraction { get; set; } + [BsonElement("IsNoFilterExtraction")] + public bool IsNoFilterExtraction { get; set; } + [BsonElement("failedJobInfo")] [CanBeNull] public MongoFailedJobInfoDoc FailedJobInfoDoc { get; set; } @@ -64,6 +67,7 @@ public MongoExtractJobDoc( uint keyCount, [CanBeNull] string extractionModality, bool isIdentifiableExtraction, + bool isNoFilterExtraction, [CanBeNull] MongoFailedJobInfoDoc failedJobInfoDoc) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentException(nameof(extractionJobIdentifier)); @@ -77,6 +81,7 @@ public MongoExtractJobDoc( if (extractionModality != null) ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); IsIdentifiableExtraction = isIdentifiableExtraction; + IsNoFilterExtraction = isNoFilterExtraction; FailedJobInfoDoc = failedJobInfoDoc; } @@ -96,6 +101,7 @@ protected MongoExtractJobDoc(MongoExtractJobDoc existing) KeyCount = existing.KeyCount; ExtractionModality = existing.ExtractionModality; FailedJobInfoDoc = existing.FailedJobInfoDoc; + IsNoFilterExtraction = existing.IsNoFilterExtraction; } public static MongoExtractJobDoc FromMessage( @@ -114,6 +120,7 @@ public static MongoExtractJobDoc FromMessage( (uint)message.KeyValueCount, message.ExtractionModality, message.IsIdentifiableExtraction, + message.IsNoFilterExtraction, null ); } @@ -132,6 +139,7 @@ protected bool Equals(MongoExtractJobDoc other) KeyCount == other.KeyCount && ExtractionModality == other.ExtractionModality && IsIdentifiableExtraction == other.IsIdentifiableExtraction && + IsNoFilterExtraction == other.IsNoFilterExtraction && Equals(FailedJobInfoDoc, other.FailedJobInfoDoc); } @@ -154,13 +162,14 @@ public override int GetHashCode() int hashCode = ExtractionJobIdentifier.GetHashCode(); hashCode = (hashCode * 397) ^ Header.GetHashCode(); hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); - hashCode = (hashCode * 397) ^ (int) JobStatus; + hashCode = (hashCode * 397) ^ (int)JobStatus; hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode(); hashCode = (hashCode * 397) ^ KeyTag.GetHashCode(); - hashCode = (hashCode * 397) ^ (int) KeyCount; + hashCode = (hashCode * 397) ^ (int)KeyCount; hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); hashCode = (hashCode * 397) ^ (FailedJobInfoDoc != null ? FailedJobInfoDoc.GetHashCode() : 0); return hashCode; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs index 9908d12ad..9ea387d2d 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs @@ -84,6 +84,7 @@ public void CreateReport(Guid jobId) private static IEnumerable JobHeader(ExtractJobInfo jobInfo) { string identExtraction = jobInfo.IsIdentifiableExtraction ? "Yes" : "No"; + string filteredExtraction = !jobInfo.IsNoFilterExtraction ? "Yes" : "No"; var header = new List { $"# SMI file extraction report for {jobInfo.ProjectNumber}", @@ -95,6 +96,7 @@ private static IEnumerable JobHeader(ExtractJobInfo jobInfo) $"- Extraction modality: {jobInfo.ExtractionModality ?? "Unspecified"}", $"- Requested identifier count: {jobInfo.KeyValueCount}", $"- Identifiable extraction: {identExtraction}", + $"- Filtered extraction: {filteredExtraction}", "", }; diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs index 37fbca4cd..730167c8f 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs @@ -1,7 +1,7 @@ -using System; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using NUnit.Framework; using Smi.Common.Tests; +using System; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage { @@ -49,7 +49,9 @@ public void TestExtractJobInfo_Equality() 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -59,7 +61,9 @@ public void TestExtractJobInfo_Equality() 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(info1, info2); } @@ -77,7 +81,9 @@ public void TestExtractJobInfo_GetHashCode() 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -87,7 +93,9 @@ public void TestExtractJobInfo_GetHashCode() 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(info1.GetHashCode(), info2.GetHashCode()); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs index cef7f9de2..7c56cdadb 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs @@ -1,5 +1,4 @@ -using System; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; using NUnit.Framework; @@ -7,6 +6,7 @@ using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB @@ -60,6 +60,7 @@ public void TestToExtractJobInfo() KeyTag = "KeyTag", KeyValueCount = 123, IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); @@ -74,7 +75,9 @@ public void TestToExtractJobInfo() 123, "MR", ExtractJobStatus.WaitingForCollectionInfo, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(expected, extractJobInfo); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs index ec302168a..d7e89e68e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs @@ -225,6 +225,8 @@ public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() KeyTag = "StudyInstanceUID", KeyValueCount = 1, ExtractionModality = "CT", + IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; var testHeader = new MessageHeader { @@ -254,7 +256,8 @@ public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() "StudyInstanceUID", 1, "CT", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(expected, extractJob); @@ -444,7 +447,8 @@ public void TestGetReadJobsImpl() "SeriesInstanceUID", 1, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -522,7 +526,8 @@ public void TestCompleteJobImpl() "SeriesInstanceUID", 1, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -631,7 +636,8 @@ public void TestMarkJobFailedImpl() "1.2.3.4", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var client = new TestMongoClient(); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs index 7f75cb7a0..996f7d3c5 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs @@ -26,7 +26,8 @@ public class MongoCompletedExtractJobDocTest "test", 1, null, - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); #region Fixture Methods @@ -81,6 +82,7 @@ public void Test_MongoCompletedExtractJobDoc_ParseOldFormat() // NOTE(rkm 2020-08-28) This works by chance since the missing bool will default to false, so we don't require MongoCompletedExtractJobDoc to implement ISupportInitialize Assert.False(mongoExtractJobDoc.IsIdentifiableExtraction); + Assert.False(mongoExtractJobDoc.IsNoFilterExtraction); } [Test] diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs index 806432297..61a46d786 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs @@ -62,6 +62,8 @@ public void TestMongoExtractJobDoc_FromMessage() ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, + IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); @@ -76,7 +78,8 @@ public void TestMongoExtractJobDoc_FromMessage() "KeyTag", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(expected, doc); @@ -98,7 +101,8 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, failedInfoDoc); var doc2 = new MongoExtractJobDoc( guid, @@ -110,7 +114,8 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, failedInfoDoc); Assert.AreEqual(doc1, doc2); @@ -131,7 +136,8 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var doc2 = new MongoExtractJobDoc( guid, @@ -143,7 +149,8 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", - isIdentifiableExtraction: false, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs index e45ed7b39..3e22c8924 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.JobProcessing; using Microservices.CohortPackager.Execution.JobProcessing.Notifying; @@ -8,6 +6,8 @@ using NUnit.Framework; using Smi.Common.Options; using Smi.Common.Tests; +using System; +using System.Collections.Generic; namespace Microservices.CohortPackager.Tests.Execution.JobProcessing { @@ -71,7 +71,8 @@ public void TestProcessJobs() 123, null, ExtractJobStatus.ReadyForChecks, - true + isIdentifiableExtraction: true, + isNoFilterExtraction: true ); var opts = new CohortPackagerOptions { JobWatcherTimeoutInSeconds = 123 }; diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs index ad1af688b..b5f562564 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs @@ -72,7 +72,9 @@ public void Test_JobReporterBase_CreateReport_Empty() 123, "ZZ", ExtractJobStatus.Completed, - false); + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var mockJobStore = new Mock(MockBehavior.Strict); mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); @@ -96,6 +98,7 @@ public void Test_JobReporterBase_CreateReport_Empty() - Extraction modality: ZZ - Requested identifier count: 123 - Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures @@ -138,7 +141,9 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() 123, "ZZ", ExtractJobStatus.Completed, - false); + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var rejections = new List>> { @@ -194,6 +199,7 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() - Extraction modality: ZZ - Requested identifier count: 123 - Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures @@ -248,7 +254,9 @@ public void Test_JobReporterBase_WriteJobVerificationFailures_JsonException() 123, "ZZ", ExtractJobStatus.Completed, - false); + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var verificationFailures = new List> { @@ -284,7 +292,9 @@ public void Test_JobReporterBase_CreateReport_AggregateData() 123, "ZZ", ExtractJobStatus.Completed, - false); + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var verificationFailures = new List> { @@ -368,6 +378,7 @@ public void Test_JobReporterBase_CreateReport_AggregateData() - Extraction modality: ZZ - Requested identifier count: 123 - Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures @@ -431,8 +442,9 @@ public void Test_JobReporterBase_CreateReport_WithPixelData() 123, "ZZ", ExtractJobStatus.Completed, - false); - + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); const string report = @" [ @@ -493,6 +505,7 @@ public void Test_JobReporterBase_CreateReport_WithPixelData() - Extraction modality: ZZ - Requested identifier count: 123 - Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures @@ -539,7 +552,7 @@ public void Test_JobReporterBase_CreateReport_WithPixelData() Assert.True(reporter.Disposed); } - [Test] + [Test] public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() { Guid jobId = Guid.NewGuid(); @@ -553,7 +566,9 @@ public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() 123, "ZZ", ExtractJobStatus.Completed, - true); + isIdentifiableExtraction: true, + isNoFilterExtraction: false + ); var missingFiles = new List { @@ -580,6 +595,7 @@ public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() - Extraction modality: ZZ - Requested identifier count: 123 - Identifiable extraction: Yes +- Filtered extraction: Yes Report contents: - Missing file list (files which were selected from an input ID but could not be found) @@ -594,6 +610,76 @@ public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() Assert.True(reporter.Disposed); } + + [Test] + public void Test_JobReporterBase_CreateReport_FilteredExtraction() + { + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: true + ); + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobRejections(It.IsAny())).Returns(new List>>()); + mockJobStore.Setup(x => x.GetCompletedJobAnonymisationFailures(It.IsAny())).Returns(new List>()); + mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())).Returns(new List>()); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: No + +Report contents: +- Verification failures + - Summary + - Full Details +- Rejected failures +- Anonymisation failures + +## Verification failures + +### Summary + + +### Full details + + +## Rejected files + + +## Anonymisation failures + + +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } } + #endregion } From ed7f502d7236c0b6dc449d70b2a6ac60c387dafa Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 28 Aug 2020 18:51:09 +0100 Subject: [PATCH 074/138] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aab4f07cc..df9f219ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - New service "FileCopier" which sits in place of CTP for identifiable extractions and copies source files to their output dirs - Changes to MongoDB extraction schema, but backwards compatibility has been tested - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated +- Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions ## [1.11.1] - 2020-08-12 From da3be79b87920489c537199fc7b59f9d0cd3f3dc Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 31 Aug 2020 11:13:55 +0100 Subject: [PATCH 075/138] Changed mocks for testing/null classes --- .../RequestFulfillers/FakeFulfiller.cs | 6 ++- .../ExtractionRequestQueueConsumerTest.cs | 39 ++----------------- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs index 41a9cfa20..66461ba70 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs @@ -7,6 +7,10 @@ namespace Microservices.CohortExtractor.Execution.RequestFulfillers { + /// + /// Fake that automatically finds all UIDs that you ask it to look up and returns + /// a single image name for each requested UID. The filename will be the UID(s) you asked for + /// public class FakeFulfiller : IExtractionRequestFulfiller { protected readonly Logger Logger; @@ -17,8 +21,6 @@ public class FakeFulfiller : IExtractionRequestFulfiller public FakeFulfiller() { Logger = LogManager.GetCurrentClassLogger(); - - Logger.Debug("Faking a filename"); } public IEnumerable GetAllMatchingFiles(ExtractionRequestMessage message, IAuditExtractions auditor) diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs index 56afa59bf..4656bd4c9 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -70,39 +70,8 @@ public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() private static void TestRoutingKeys(GlobalOptions globals, bool isIdentifiableExtraction, string expectedRoutingKey) { - // TODO(rkm 2020-08-28) Why do we need this much boilerplate to test a string & a bool? - - var mockFulfiller = new Mock(MockBehavior.Strict); - mockFulfiller - .Setup(x => x.GetAllMatchingFiles(It.IsAny(), It.IsAny())) - .Returns(() => new List - { - new ExtractImageCollection("foo") - { - { - "bar", new HashSet - { - new QueryToExecuteResult( - "file.dcm", - "study", - "series", - "instance", - false, - "") - } - } - } - }); - - var mockAuditor = new Mock(MockBehavior.Strict); - mockAuditor.Setup(x => x.AuditExtractionRequest(It.IsAny())); - mockAuditor.Setup(x => x.AuditExtractFiles(It.IsAny(), It.IsAny())); - - var mockPathResolver = new Mock(MockBehavior.Strict); - mockPathResolver - .Setup(x => x.GetOutputPath(It.IsAny(), It.IsAny())) - .Returns("path"); - + var fakeFulfiller = new FakeFulfiller(); + var mockFileMessageProducerModel = new Mock(MockBehavior.Strict); string fileMessageRoutingKey = null; mockFileMessageProducerModel @@ -141,8 +110,8 @@ private static void TestRoutingKeys(GlobalOptions globals, bool isIdentifiableEx var consumer = new ExtractionRequestQueueConsumer( globals.CohortExtractorOptions, - mockFulfiller.Object, - mockAuditor.Object, mockPathResolver.Object, + fakeFulfiller, + new NullAuditExtractions(), new DefaultProjectPathResolver(), mockFileMessageProducerModel.Object, mockFileInfoMessageProducerModel.Object); From 0b5db87a56cd553ff727f1f2aafb44c716610fb8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 12:12:45 +0100 Subject: [PATCH 076/138] Update extraction diagram --- .../Extraction-Pipeline-2.2.1.drawio | 1 - docs/extraction/Extraction-Pipeline-v2.3.png | Bin 0 -> 113763 bytes docs/extraction/Extraction-Pipeline.drawio | 2 +- 3 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 docs/extraction/Extraction-Pipeline-2.2.1.drawio create mode 100644 docs/extraction/Extraction-Pipeline-v2.3.png diff --git a/docs/extraction/Extraction-Pipeline-2.2.1.drawio b/docs/extraction/Extraction-Pipeline-2.2.1.drawio deleted file mode 100644 index 0e0121bb4..000000000 --- a/docs/extraction/Extraction-Pipeline-2.2.1.drawio +++ /dev/null @@ -1 +0,0 @@ -7V1tc5u4Fv41ntl7Z+IBxIv52CRNN7tNmzbdt093iE1sWtvyAk7sftjffiVAtiUdbLAlMM22nakRGMM5j867jnroarZ6FweLyR0ehdOeZYxWPXTdsywTmTb5j46s8xHPMPKBcRyNiou2Aw/R97AYZJcto1GYcBemGE/TaMEPDvF8Hg5TbiyIY/zCX/aEp/yvLoJxKA08DIOpPPpHNEon+ejA8rbjP4fReMJ+2XT9/MwsYBcXb5JMghF+2RlCb3voKsY4zT/NVlfhlBKP0SX/3k3J2c2DxeE8rfIFy3fQ1LxeT76vQuMxsa8+px8ufCe/zXMwXRZvXDxtumYkiPFyPgrpXYweunyZRGn4sAiG9OwLYToZm6SzKTkyycenaDq9wlMcZ99FCPn+zQ0dx/O04K5JbyM/ffFCz2GchqudoeJt3oV4FqbxmlxSnHUGBWULaCG3eJmXLaMYjCY7PGIMCQpojDd33lKPfCgIWIeYdneJ6Zlm3+HIadl2v22Cou4S1GFo2JDTb5mYVneJicwBT0wLtUvMgdldYjoeLzdNw2uZmEZ3iYkMVyRm21JThubD3S0ZeLtK42CYRnhODu6jRTiN5iG90OpbffIld0qe7PIxJp/G9JNlWMaFgcg/iRuEOClP8iSN8beQEX2OyZ15PhRDwTQaz8nhkJA8JOOXlNQRsbfeFCdm0WhEfwbkMY+CU9hXsAtglQfwCunilSyTOS7dXicyXz5mXMNPPfRGPnlBTj2kyxF9x55FSGQQm7xfcl0YRyHwA/Tc7YwaxiLfD8w7YuIu6JnZaky9gf5jkETD/ggPlzPKIgWzjVcDviGxj2m5Xe752maaf1hwjQlmF9VffePJBI/sDsZekriCnQHQxITEj+foIkoFl6KEKPuJfDyp2qOFK9Hi6ss9GXhDxOF6FiVEBIq0YZNoEeNhmCSHtd1jMPw2ziTjx2WayfR8vCp9nRL6lgpNE5CaEFldbfPOk8j6S/AcdEZNncqH4uxAnuoIYIOpDd0DiQ13wXz9athQTAcDMPhsiA9IlxFRwRnRrYZMl3cqIDVkNCp6K8QO6qghBaRqjRYyKQorE8dX72/b10CohLS1NBBEUW0aSNbrP4ACqs4GFjRsWQHJZgB5XuMngun/vBpWsIhYVSWka0Yg2RhoXAnZFu8LIduW5USjDoAlQ/QkLbShcgedIet4hOynbRdpIUcOrvAExykZuyfaNBifgWdolYHtfD1Dpn526dqzuhPAPJUPZ+IZItkj6b5nWH86tO4Z2hUy0tqVMvI5pTyQSDJoUkLYiv3CDYmVaKFmSSELy7v1w6f3pYpnuCZKZJRNt0OaJ59g7x8VqSK7tm/C1A6zj2WJ6AK0trXR2pJpHabBKEiJ12hcX3ZbNNZgzwGwa8uK2senJbT5J63naixZApwmDMuCpx0wyU15hp5Eiw1tO0gL2VPLvZNN2LBURTQWM7RKqHu+vgng8nXeNanBhjNxTcwf0TWpPxvad03KUuMPwzhapBI/2i498UxXKJC1ZU+GVSBzFo2vi4JnUX1icTSBI64DmSq+ttDD8VTZT2VFahyghT7zTrFJg7pMC9X54DKR2wVaQGGhogqQFQGyCqUgL0UUawR/J1qOOK/UEsxPkQfZfFcibOORayUZZYgn2qxDS/ZPu28e1uADMw+d/sDc/SuxBVl97gKASdqMR0s2WrpvPNafLKDxaPV9Y+evCbBFmy0p+6u3ye2IvHb0FGViWGRQreUEp1oKpWC3zL7t7P5FnDHlyVT2nP4BImsLoDlnYG7ayBONcEc2OKG44kAbWRRn+B2lNlaj+QRHcTTRUZrgb5YUx5dk7idtB0mhuOzDUSZNxBSRK0mSRlNEjuyt3eH5GAPJofYTck71ZHhr9Cwte80XV3U96VaqKUoBf+HxqXgHsOMaTcq5FTRGOB+9oY0cKHGnQZJEQ55D4SpK/9z5/BelInmx/Oh6VRA1O1izgzl5+j93D3a+RQ+3X8uO1qWcKSRPgpfxMNzzpgUW0yAeh/uYWkyrcMS1pdg7qRyAY2wsDqfEgX8OuYeF2Fj8wj2OaLR0Y6Dagohk057dIn/v4ltbMMg38oUbGcKNcsJIN8pQtXntE4BWQR//GEAbVAQaW5p2JkizHUVIc4yWkSZr8fpI26BmC5S/OJzAqGEI3aLyrx28wgjN1sQXj2aXwu5MUOLZvsjcPmH45o95HGY8ZlNuOskYfaP8troRVCFSXVVWGcfIKrMpWcXi0AeFFcuhnQkMbcFzQIZ9HPBsV5R6wo10Q61Cfeg5CSv9avHMkIY8oYuQebRa5G9kug2rxQrld51DGv50uUDfv3735/8LvEEaz975mLVf6RrSPNEAMwSnryrSkCjTRMjqRpqco2kMaaWo6RoaLIGJJnKOQ4NttYyGClkLEA0JebC0mo1VQ6woB9ZBxcc8+K5FHhCvrxzr2MjDgL+R7TULQO/YENePAkAm2Q4CkMW/zwSBSESgaJtXVogCAh3RW9CNQIWxr1pQIwf3YRyR56cR8bbkn1sRfmcWEHME+LlHW/4i/Bq2/D0oICYUm5H/gxlNhGTD2yzN5/DvZZikm9PsYqEYDapBo03sfg3XX4Ix3ODufbRz3+x83myP+yFhlmzTNObhjJuGZpOWz8epTNZdm+uD4suY1JZj86BYVRXm3kS0KgdiK19ICLHuOhriGb3BPe2xDV5Cqxe3V5wZGx3RuAbYOADK2rRxkUmI6lyk9H1YDrNiziO5yLMIuuLnIJlchk84DsvPv3lKgU4JJ3F4NyJuKGC3ZwiNtVm0ebdtJVDJJdob6tgNlSwdZPfvRJU/RYSoR/L7PqBYufkjiOfRvEQof1xQqR9Me87lUxDRF5kl455z3TEEqZARHg8aZAOggercBrpAA5Xx7AUNZc0Vnk7DTJXfzp/wsdAhevz37GfBs29XC/ITFJjGE/nJkoa20sN+wPOLMFdGtGDov5UK2duGhW0JsgRshA6sQnJ0lUT7kHexFxeFUXciIEoNOwaWK8IqeRlZ2wxEJm/CWQgqpgZZqE39m4YCD1Gk1VHVEns9Rs5f1JmV7FVzGIuZdyb+otDUHx3rLppiwLZhd9E0ZH/x7YpICQpEoOG5Y3yjTC1amQugPaKer2ch1725cd2e2qI+HTYCEorhHVmSsCLIRrrZm4bsDlJ4Gz8NJ8v5N8If2rbeMkzjm5IGjR3iFTIsgVnAQviBKzNL3zYRhpwo+8e5o9zKeqne5HPqtTHKcXk2efLmKB7gm2tkk+yt/WO8Mqa4htMXDV/IbrKg9ff6OCO7RNkEelWs8VwkMAbaNgwBakgjY+Sk7+tjjCtU/KBtA4vdGdOoLDPlZGhmv70qxthCLbzpyyqmWUEG9gMQeKJ7aafl9f2dPwPbE4gEpTocrkzXAIKolq0r8OFVi6JmAxRFHDndv5eYnbhIMny9oYaxtVhtT7K7/Bqud1wf8rD57cpzXyomlIZ1TYfn0gF4Vm/+6PFl4Y4cJfMa1Uiqu4+U06T6xCvvjFlpYlnaJhZUwFclp3gXJkm2a9lx2WJuceBvt9claYwYf6UbDVOyobKA9M6NXnBMAxSjKIYv/gU/UhgsH2dRmgexA8VRy7LdECtDrXYTD8TNPoP5S5t0py8hzofa3+gzQ4AecAw7xjOeLmdyx4hO2SSqGGm5/DoYU1TKbQecVO/yU06j+gYNr4JMBO2y2mRTTxPYNu00YpVuU6JuTyRHH3Rko5duiPk8pELgQwCIgMZ7nJplDTBKlXnVjZHEcm6FZJUXerwP5uO+CoHaYoPNGqzYNIyUWNFwo1OgwyaYE/3ROXHBR6gQ1L0K4ow+b0BxoxwVWoulkEA3G5DLurqamlU21W1oN/AD3KuxF0Hre4IDe3UecKq2FvE8U4ZHVn3FeEHoRIlhHrzCkq/o92XNobiG42Qus0xRxZljaRMrFdZY1po5vBogcye3ZWFdoKPeWQiUDjxoHlmQfYM26VYNO4vK4tuUCN0pf7F+Otv2xYScAUSxG00uoPJa9GQRzHtQ/DUj3kWSU49GYOc4ngVTOQZ7F6SEDzNMFw0YHzBtLDnMWuMmO7Iw/xkVpYNtz7zNcju2as6vPPU0MljByt6yMjWj7/0Qi+g2s+BMytLEtQ72sX19zAF/I6fhvj4msMHptro5AWvTqHlzOx/nK5jAc8T4Gcf5Igrg/A2OP4fPUfgCn77CtCouCxp3WcsjZyDtrCDnJDwgB6EvQgzuIiCYy7ez3DxOltR+zR2sEhBUWLF0m7zdFqPXrEXvAJMdMXQsaxMfqH5TUfP8bvXp898XSXjz/stgsDZH+NP6ihX5lvWGjMNFthmrQPK2NyGxhFQKcoCepQAZWY2OcjJ2rFVVdiRWmKvVyvvQdiY6GYntq8Tm2Me2ekFiokedTt7XamevvzvBs8dlhVi9gtnpIVtQZMRogWqugBW6LHqgft1W7RW62bK6gCir0bFBH/rtz2GQQBuH0PMPaZAuaTU4Xd2nObpTX1MNxOWa1TrMq1BVIMpV9GjrRKcQE2jVsG/en4k4dQXPREp3V25DicR4lzZxmnxNP/96+/W3P16+Ps1mlxgZX2xQnNasXzOg+jV2l4gNZKvHCxkgyoe3q+EkmI93TeBIvJFUAncw9kZ8rOh7kQOhIA2WKU4KuVEr2ragxM/Y4VzShcmg/QxG+qBsF5CvkSZOuboROraZDrBUQZOgAvEj29Q68ZN3H/i0DJeKsVIdDCKqZHAIOFPAdVHaIAMq6wZ3NlURmQMZX16wp5jxTDh8jDO+51UaKnnfXuL7NFAIQTa2ZvNAhlsbIMpLozVIgtxWfXWCwGGJcmZ2DEBBAFX/aON7eZJZMd/l3hJ67IbzhgBiiyI2TX3BLE2jU788cKpj6hMYdI7xMU7zTTXRte0okgXIEoDQtiQAVnppwkHRTuRfGNDmui4HA2b9t4eCxlzKvDtVnpLPXct/AUHsQpsvfbSBPcubVQ+bneqbkguv1TbY9G3Zt4CyUS8RXKvRkHX46rwDW/QOPGDm6yrfgdlf3mRSz8zvFs81SH/k8O4BYA7o2u0PRgBUn6rVN+AgAK4AfXWgcEVXAUhvQY0l9aGisejhjkXQLdmgQhaYvDlgeYA50Kxv0FiQUPYNusV+LQEDIWKMADgoMg/IYYwphzbn3tHqqDs8CukV/wc= \ No newline at end of file diff --git a/docs/extraction/Extraction-Pipeline-v2.3.png b/docs/extraction/Extraction-Pipeline-v2.3.png new file mode 100644 index 0000000000000000000000000000000000000000..6addc880ddbb8a5c5baa14b12b4ba46f484b85a5 GIT binary patch literal 113763 zcmZs>1z42bw?3@Ayo!x2HYzqc9R@Pp-GhqL-90vn9f*nwDs}@ZC8*d4Dq?{O7_TA* zVqu^d|2@9`&N<(A&UJCkJUq{y{p`K=+Uvg8z4mf0o6@CY|Bh|iwCO^l68UY~{8iYd zP1ur%cHquBr1?ynHvQ~wlGJVSYYk>in;1CZ->VoHL~nDsW8lOX7>s1rsN7DQ6gq#2Y9gLP3tZJLj z1uoe_@BxK@+i*O%0uNA#e=q$g#29b~XSbU*5{;5!aL2&NNH}@|90hKVW>AGJW(K5zpK zGxP9zI#WVG<8&;m+hX_FTx0@Us76^e@SvAS#&}JjokYn89eYV;D@Ma)@ts^N3+_kC z-EO^%P6f~HJeC(JBRWANjTFZvxh+UJ0ZKvZ!Ag*A4ku46U@PrxxtBxs1leqeNuYLW ztptVBtn{-vT8ho!X96c88kZo8X~r)nW?U=F;5g`~KR;6mvQ zfMzNo9em0u;0DagBw$>&pe(2sGuWX;<};~K2a!YK0(T?p3<4|%11>0Dq!v%|GB`e& zl;yNK^?WHRWC|w2Eb(9vcs19HGfON1r&}!5nw6kupI@T1tBn#SN(;9UnOM+RV+w-T z<0%NWP-7L-5p*@ntB^7MT9X+f1rM|&oy34}LrfAS+2uDHS+=Hc2I4)9~2f=#wN z+**o+A6gVf5PYZ|Ix>MkFmudsipDF)D;+u&Mo&Dip^K~eXj~jsSd^V#+rJ`{4 zMxPB~V|DWV`B7FFFME#K#^dg zc)5V3v?{nhD%m7J;LJ9a9)+?5{Y)-K#qv5lq97|k^x_>1vdfI&OOYIcKyPGuQ5u}l zrq_5VXq1Z zipgwP5QYsoXlUux7?Dr{=i*HSvWrjDvQ;<&QQ)HJSUM4lpfmfW45Uj-W}?+51Rkyz zgCIs`%1IoA39UEqy<8`g4yRZkFm#Y<;o9xssTKGu%FTi>)n*R|1%q(Gi=a}8oJYYB zd<=;kMyCb{N-7phkt*2y0FOsiqikjz+~Skr7)Y~DPd9^Ti|1J+PMC^j#W{h?;@BQO z4#E;qYz(*F0~fM`1SsCblbb9u2?}9lDeyJ}9xL^_t;|4>ta3u?2V*Whoh)PVpj?XsXAfws0<(-m)|e@L44)v9V^K5{BEWJ4#0E2+t>sV!5+z!s zCi`7%zDY~9J5)LcimUL_JY1m&NmdG&YA%^?WiqTZxgPCN%3UUu8HLB7nFI(5?Ka5h zO0rGvMVSn4q5)}EC=qlCGhl}b0(OOs<024XJe{5(gV`B=m^wg3!`vdciYBti=@_E} zBUU?!I1EB(M?=IowVK5f^Ne3`+1bRRS!Zw-YBIASviUTDVNWuPg z$^&8_R^i2R7#OSAMpY1fcsmK};NnpHfRP+Ti$uU*g&|wgnFIy+4)Ap{xR(Iu5pZxD z--Bf7LBPSg^-!B%Nx)l0Mvb2VVgQDTW6JPim4)xt2Ax5zgr%ig113BlBf<&MQk^9r zapU;{mDB-3tAay82LXEFWg@&7d{L-kidAC5TkRCS0Y^6JWNr!#1)(cQ;2}>yF!^yd zyjtpJb2NA?L(IY|?R=paX7s4tYO9^ELBfe%y39iM7-2TA2t{x}r3juEZpXtFW&xOk z$}E-|EN~uF=p%VSmrSaWAVHy}Ziw7W5kM^_kC6)3vUn^JXzU1RIa(n~f`r)^Bm$SL z^4q`*bWjaM!_)<=G_sp)F+u!D4@=7xXpv~(y!a5jF(^SA#bwu;F&MFfX43)y#s@)| z58BWeh}Njk+Jac9(XYTriEgxC$j}qv4xv)Xa!?px+I*}DBiA_qw*Wu{VmsN31NM+X zEf^YCsSfRHM36@EiBbL!| z7KYAXlsaV^r^Fy~&`>O~Uaw|TtYnu@O<^FAP6Ju4f&!k0cPq$3Gs;b4aBO@Dlgq~F z1!6TDVI@=bcB4w*17rkmC5Lt{j|ijljZ{)VOLQ6qa-M?8cX8AT07e$F(GwawhN7nu z6gZ1p@4$IIMxIRObMfIc8BR`RVU1QDXySkp*dQ=yaYBh0EFVcp#CTu~v7V2h1_d0Y zlIjY@3NnKP^WzOPBT1$wQYkJOxMkq5*kCL&xk`vux?E}q+XV3lg-U|T1*1BhbfXro zKw=#%gaU(SA+=VO$qsfhhffmV3<5U6Z!r3iB8Y_M3uw@Ef=%S6NN{$!2F-BD5GWBB z2c?>wXc9{jdOMD7vl}277Cz)P8UM(jTj-2FJ2O z+gR-vh*c09m+W;41Uwm*gOed#5F5n-$4i7X4^I{#8G|&eR_CCI1u858ju6A?TE5;T z#S_^9j)&vN$V6@dh9?c6*lr(E!52!rIMA^`O+X5WRF>B4r@&Nn36H?WzDBxiYrR z9AaS}tAq&Ep?pG(4;P>-r5L$W*=xQj>)iGu5b~%S{*-TpJp&O610R(Q-LM>UF|tE~=UeRpM1jBnA;mj1a;g z1tLUpIbs4|=_IrDF1IcKR@o|JoA6M!f@H4c+ffnZUCYMPfEO0L{CCfJ}Nv((^*YQ#E%8zNNTWo{$g?gCGBUJ=@c*CGNq zv|8=8DJ)(F$tZUrC=LsY4S6W}79zVPIO8+C;N*4QLO?$#4byCX1R)aH`k{ z790Y8ERkU(0TD!MvFliXNR!<(otY*e@i;yTN2DYf6*8!sg$}xCHYeRLGnsT$r3(tt zGXx~y4J;f3Yt+M#S{+BnfeO?vC7W-x*&$xJL+X(Vb!-)#jMM{UW>KADm=zWR4xiZo z22Iul{3<&UkOgwc6=ZI|+(NU9=`uYACB)*WT9U|OHa3xBN8;Ez3=RWT@!S@PnSz7ZnM^9eu94Bqp?HT;qse%i--New9eR#f z5WoYYk>O0Qj7M;g)fB&-Wo03SD6)&~@?g+JvKnr|K)FN_g>9F4u>>Q|s1FcjA~DO# zbm|;vip8ri>1=4F2`hI)-5dtaW^*8E0;9zbm>-NpBqKa@xmind8|VQvm*s%-IeZos z#T9!AW-rM~rdZ5+p_XokIcX>x(`DA0)nYaqMu7y?Y&Knl2BL;mYUbJ)3Z~Z^#QK>M zvH|aL1dytrUd{`0N&Eo8N0eCPG(ebzG=ooshoPZ7v>0Wlvju#L)&({j9)=OhJaC1_ zOVX*da4HOq5>k9Bs9tUcxj&vK#n1x~u}vitDL4VB7|rqdj8YT=4x#XP8Y}|`q9|Ex zpa=0Z3X(#?LFGDvmx0FO9cV;Q&F~nZG$miE7Kx2GG*tq81(;N1^xHKoh04PZ5da!h z=3|;&GO$7f7DF%AlfjnsJK<^rj1IMn)FLQR;=^KSa07+s7GW`3GQmQ{vOQFpizRS@ z{_t2QMFpY)0xb;k zIC_E@i8WGHTE8DFCvfRzi%f-6STz{6Q%`|u$pVI1>(tXxIEROUhIr9JxYmYub3+I& zfU1}t4~?pEAwX`5SIRA1xJU#=0ixoyO3YS`Jw4 z7(fU0bS%N+BjI^oKT9npO0`5Ghv$SLX?zDvU?tH4M75F6vy=5E6;v$bGeX9&v4b>+ zTWZ4@B}xw?$W~xIR;QYbVh4S2g%6L$+gv0r2a7_fTwW=Oj^)v{cBd|pau42=p&wy>ls4b-bu2kj=f1{hdJ zQjk$%jW;OpVC_nWfkg(ufaYbF;KXPw878m;oV6jeP^XaXClXZH zPzyfJtoBK1K|4l|;wya|7eoT^k0`Vlv^o(*#x$7aNT=CIQt6EhVi4w#qm5Rt(8A** z41BDW2$8cX4uTHFvZIXxh``SwQTb8?uuUkWIN55o-^BEiT|pAkB*KJPg4ZQ-@(c)y zSz@4RY;YPusHB@jMxCBSciIg`f?FjLfy9`tCQE5DwZ)3WAkAL8#6qJ`334GmK*#f; z1`-#`zCLtV3165{ID!h6G2`7aZtw9Y+0A4{r*)a$e-75FO zEqo41i|{kWLZGU!1$vO=m>@Wp8AlW-&`zOF>J5@qZWu%e)M$!|!viw_AQI&H0s$77 zyVz^TQ4J!PT?*(H*%|6w#?sT-5H?%kGjh=Yo9zKL+hkRW)i9F_Y1CkNn2_2H2cd)f zABFq3+ztN!Q(Sa)mLA`0{H#t& z9O?Cn6$hj=BF}uR@7bjOmzF4(BfWR}bsD>?M^D7Pqq8pTSk{XaKSuf2)n8wkAGZIh zQ+(O_z~$x4eI9t@dOKC;z2ejQp0`&2YHccfG!HKC)s__Bog4oD`&!tM{P@>~I`O+# zFK_hzzgmnBqgy@0M~AIgFl3RdN?|4XZ`2KoHMAUL*|_byeWD< zo__Le$;qN$KU*tmOgWagpDij^S@hD)mcj?s(`Ps3@53!>S8%=CwC}UnlbaCDuMX#= zLsg}lwH4`G*0U#-cODW~=YDrzl(*yt0)!y;R zeF{(i-Ko!rn$IztrYo0p9VFK+OL_{rvL2iCWZ7`$+QCzm9Kz9tpq<@g{n2x$K3o~6 z1?H)o^W($SiS5_S_p3V^1`64ChMTt}CEmEC%K&Yk3DfoD)%`cFuVZh;9P5f*b3DwD zT05fb_~iD>23(41e2kv^?c%w--ukXzj%eoIg!fm_j>UZMf zXirq%*|Q#>UFc1GH}~G8laEe+s!mwgbZOB3YRUgv(i-B>lsf#-=$uI5%{?9dNj|WA zpS>}w5Y?D{(mE(?itXW)=5LKT2`uG=3QND2@0SmlySr`Nr#pgJ2R>s}yI#uJ^qaq0 z&-}_I8eiQxxU+^N*^SZP7|MV|E3C8c&hX73k9%z#dvI-9dQxe==4#rOk_&kixcW3y z_a4)|_caZtny=@kC-kkn&6)O~MSjiv^c}MB+K?>Nvz&%bOx%G!4omY3vVEuE#@sxSe+jOME5gU0`;Ny#%yWx+D_S98%p7-m=7Y}n! zvefhT;Z)VF#`Kux!o1VZ(h{^0O;Nh7ieLNU+uX!#_di*_aAZENUL3dHyLjuZy9?V7 z)^{kJ{ALSWrw#97smYTm2IA`b+*VXq3_>>*KV5jS-Rgw!4odpqeTJ?*7KB?0AQf2? z&!+sE{rUBl?rHTkr@o#3U*0o1EGfQv0bdc9hKJnTiq}WuH%xwW;zEyO@X9Ih9z|U~ z6nV-KKX&&ft*0fneEwEneHU_)(P)g0-l=EK7@~N!T&kPz6>kubnp`luDy1&WKYWI4#PI) z$c}eLEnD!cW1D9^>Cx-gPW>$1@m43DtzS7K*KiQ+tfw*t$3zMzIIm7m3R7!3MLb)d zkQlLmR*>>~Xhjy|%3SBTw&%m2w!S%O`V$a;9!1c>j^d3R=JkWe^vo3Eh?)!?*YP@SeuLWakgNxVI9#}f^?&~>MHjLRaH?o(0{N0fi`__pvYp1_>`31zp zlo4Sc%oV*mAL~&Le)cIj1&1$%9A|y+7`vDB7{uLiO>OdgzXvq7-@+p|Q!g#v__Zi5 z{rkl9g!e4V!&60{dhZ>vv*=l5lDCIT&0LXero5#}lq*KeypOLoUd z-mNDr&5jTb4g2u3tas;iIt!N$-#32DhY?u`1w)rSsacek=X?859C596S-(i@p{aRC zdMsVIVE!)ArEldqebvyOy&j#L`n(XE(_JkR1AJVOhXB zqvyykdqU_Zu+yy!v}5oy?vi8jlVN%ne_dMgMf1h{!eGv4yy6AByo@NR?n|{*P``4`}Ij6^kt%!d%d`Epj=W1@bC2D+9S6s+$ zgWGy%YJkjr**<~;a31FlA+k}{7C9Umb~X{i2`+vQf> zlA=|ki^eutcKnZFwv8#=cxK6@zy3ITM8t`DR04!pX5qfV6u+yrBTw-Vg)OJM4#Ys# zr^PaXMaxU8>KZ;2bz!vcOUypM>mg;qsbPm_9eM?CDLBo3@u$P_-B1W;miq{wWDvQKGj=I%j>kA3SOj2B?ora7mOS{WD*Tb z;6NmJi?|H`67nYkbtDWmLWt8K+f7}^6W6YzmN=H%ncVY7hc3DE6=1CSG5$j~Bc2pD zYIvLOEy>|$zP&n}_ByrWgag}ic-KZ%#y;INYyN~oMTo-syrrvh3Lgfw-&@|Ju1w?Y z*s`>Kqb@G{e*VX&>9U^D>A#|ztQQUFX*sLLjqay>R5CZ~YtZv6a!sQC})ik}|7D>Wfvpxhko`uLQ*reVHh(G>r?S&U)P z@4q)ZD4qOm$p9;au%z>`{+8%3c&X+!uAyXJa8hQSGxlaybPcQ&kg$6wS|T1sTEFJ( zf-@Hog>RncwGBLHMyp1uh&eM)!{$bB-i&c5wViS44|nAAoCAE9jdOoH*Q;UN{Q5r5 zz|XJGGrA8a_e+|0A`YgE8#$4S?zgo5)w5>a0IeDU-ZFCp@Z(do5E+pynEub(@% z;c{L@s}OPS1NN@^b{#$A=gGR5!W4J(>pPoHUHU$V;#w(^-q}@;8q1e+LX>;qtZ?I! zy5e*21?O4MBKt@Q5hKr3VxD$xe6)zxd*ZCb$&Ke5_kVWOUKW;*D#Tr^S*wn!&PsN?h@25Fute83HJ!w%p@ptWec-)7w;aNGBftd;4QlW<{E&R!%THbH3TvDFJ>HGuo zQ&C@?)qalk<25sgB(2$Ndz_0Q)?Dka{+tCpDX)posie0oSxa|=cRrpKkp39As#!F`zbx)S8~S}_YsFv zmFK$6TE|+yVx)MEDTVx{@$SUK+Iy?bv>#Yk!#xuqk$q2&N}0aWAez8BX0%@Y2XzY? z>Lhm9y9d5UGRD@*X}{{OZJ$~@=l!L=)vWO72K1vjD&+D<^IEfH#qNXopJfH+EwYk1 zMPHw7N*sSI`EfR&`upbn_@><`5<@rVpv?Df94Tr@7uOb5%T4z`#_K!ZO`JdX_94-@ z+Xa?{#=*Iz{Y`tq0_pW*6H+ek8M(V2HD*Nqzb-bm=fXoH3zaiR+>Dx$rAKAfLYBk! zG5{iOKSp~LcFofd8gJD2?K=f~Tev^Ipr(HA3C4;9VN zD+?D#MZY*A%N@Ti^vE*r;cE25d*&0nv>@bj z`77EG(+2J*mSrLgsfM}}$6h`>ee(XC<_gx`>>07^>SitKLT^{8%bct2@Oi@+!L`Y6 zPpN)P?>lj(xuDiMIiq!B+3CoKCrIU@=@(P8a#}{M++VOS)a_C*l7wk(lE-J&7k9Zz zXiV>X6=}_`ogUCVOE(9s58_6Y5${!Bt;$TTtvPA%l!gda*RW1~Cm?)}<`xFtook;} z;C_4h3+nXjFDUTR+~{~-(WV$i+UEJ+pAHEHc`Zm0#VhB$*ts-)_u9A`grx>?>)pob zq@*$V^cC}sH;flRpWNXQ!q}p$b)Rk<#{QeTj#zM(g?^x(aQY)Z_cnX-mY!_PIpef( zr+0Wp7B(^_o>osd@i;7F%_zy34bzK~W+cX6`=uU~@^NZWX^$_nAMM}NxZLzlLni4$ zY#_KOOqQG<{jxQwbTQ<|7Txw^u5k#icxTJyOUy>V2*BGd5u=N;3VK(4{gX~##6P~+ zlY=;szH0E)nv}_>YC5`9Yp$)vYR4s~j+<6Z;&LzNz5DbRuHC2gE9ZQ()EnV1N8d<( zlG&%ACi^!x*_gB-wU#a$Gid+6nMVARh{#`$U+&LDO+57u|HQH>VbRGq#fR^ka!wyv z0zJ};o}Cnir3a^!hYd|nkT6y(5#{a9Zd`NX@2Q{fsTvL)&OW}cdh?9TK*N@HaI^CU zeNgm!Rp}~d>emH<;_#}fI`+0nRVMFIi`x1rx$dF;LKkgWN^}}`_sz6Wkhx;k_$2JSDEfE(UJ+*plP=>B(&f`WeoZUp>7lnn-m#R1#&$O06}XF}df3yv-2uw!GfA-~Y$em$XrL?J}sl=jKJEse1gef~$L@5z7cm zyXFUG-#nqcEgVI%X3l<7bnWA%w;Q_E>C3Y#E!- zO^2&VEIt3_N#aU!0d4(AlT zp24~eTKd~Ud-3t*mV+sG=l$uDzGeYq`&SFlc`EV@w~v%|&l(@MXZi+^KR z;`^Li$yJ6iz&mb6a-9i?1Np;$Q;c2*u>7H>$W(7wF27gg0|?p%<`w@a)JG&gK1ay?H6=(zcQt}Y`4f1mSj zk_X5>#GXOlSNCL`Uqt^sbZ5dDh`Q}GYxw; znlCnp#Ij32;{h6(jvQMOQ?_~lx?pkClT+6xHA;uAZs@06z5IQ;DLbmu)#=R@pXMI9 zp64`&_xg1&<>lYM-+l)dU!G-+_+!L$(vDv0Fz2|xoe9@cO(jJo6C0$aV-qi(=_8r( z{0JB-ZP<=_6b+Fd`{xHM@H?u@h}mBozQw(H(xbb~d283)&nq)0i%Tv)$Bsvjzgy?N z%|cxspZ*5l@#dNigBzabZAASzb@I*g9=);pnP1n8F???xv~=eF-{EL4=)Hh1VPF13 zvAB@1eDX+V1^Vt6N87b@>sri{2v^HLODV>|$5%aqUd)Sh4oJIQJ!o-iEtRorBM1`l z-A4}qo}E^~e-Z!tUvBt{)$zB~+4r8VtxfzrPV4*$MbG3BN9xa#CNsi~ah--0rncV{ zBpz+}6^dQ2^Y6~|_Ny;_pv|ycBXpap9-c77|9By45`wpD@3MCHDh@K@YR)ZcuPenP zUZ45-DW{6$^NX}gD%U%vW$SCOar8GYu%6Byc$J?|@h1B2Us zulL3kA!L|=9aQ*lWQji)ai{;5GUTJ_k1qcWNbA$m)mJ0&&}Us>j3wqZQSj3Aj=T2` zn&dvfx?DeRWO@R%K_r>6@hiKO5}5;Vv4Wi2XV~n2lKX1Q5c36PKgKm($Y$vQPX-Gcb(xXbv1fiFW*>0|FZ?|(z$j)?lz z@mcGZ|8|^yH}H4>_r67Aa#HS^(`$h?RQd7c?NLG(BXN&nQhf^Hh*%tP(3IA>XdB2_ zN{RPJUDfkjKQ-!*CA-P14?Rt2%!q-%Y5P~Zdt*9oSyD%Doim|cRQ5Y8=NHxYP2(^7 zj7t38T+;R<{I{-;QNfN${}Q}j3t#+~ktmeUXOMttU)t3B{3QG1v+=p=nY_5*DZRVM5Qq&gkQ8~;KUW_$Bzob*OjG$sAr3AQi?0`IfhIT2a>+D$%Uv(?HJNBEL9la>K z@<7hti|_G>E2Dh3iW)K+TTlEbs;~aU+B@j(bMbfk-nn_jz22KH?U7wu8@*L8z4ZJu z@U5#ymLHx=dRn`O*0c3Ng6;Ii_m_H~kvV${mL}yi9aUU7GR^+>;c2)!Sh_0e7|157 z5nZ*kr$hGJvJU^=Bgmm(kMtMwo^#hnT+5vHXz;xdGZ%ntG5uovsH})yE8>N)wndln z@(s85wG#+wgDyNz*_GU>Z}zOuuhy`ZWevb|7}l_F{Xg^X&KE9&H&NA(>B9??vpHEa zhIpn`_n$B+@wfeVg?9SyTxoRA-^s-Q>_=b$RED+qpaDo^*k>CA8y=^Xe|rHL+iBqe zby$zdb6V`VKmpJ2+?|2$35mc@`Xg%Bz6I&I;&#d0)=7^RQNO%-AbHo?>wIJTNzRId z2a}YkjM*D^6=fZH!2NcU{9x_gAqUs|u8A!CKm>pi$X>aI-aUfY#?rJ=GSe{c=U1KU zVc>bxE???)P;4mSCLNwl?q2p67y`b`^6Tf1irmUL6Y^lMyaG@}$SitaIv}>_lCP)b zNbA(Ny+A5H7Pn*b8G5k4@5HURCMsm)*N0WrhZ<08N!dFXyWU*xn4a2I^gBNcS)ky6 z-K1L)KQCfO*0cDz4?K6bsMcUhA3vgeTi$ihq|G3=sUVNC+}xFnvOaFGWy8H zbCju_cP5wd-<){b_3*l>&DY>fx`?gUr~R4(S6+-@xW8^U^#?*j#-is_o@CBDf)%D@gURB4EO2uD-X4Ajp=Ky$FvLM zOnJXJdf643Zkwihx{C#${n=3{gq@x(nT=O^H;!e;PyTlH7O|=SsLeU{~jn zYtQmm{@E1sevd=l?%7gmRHEl_QT{5*e!{qsVVg(&`dT?3H7+YJH2|=;%f3|onrf+P zwX!Rrbu2s_Q6~NVclpw~9-sOyyqy}uFn7!d(becez3%JwBpPh-rpk94eRDN2vo_?l z?a(lLj_G}V%&dV2O?8#GtLsWB+?@ZakzHvI;=;4sK3EFKj-hca1vva>0hWhBMVc07bvI+oup#Csj+!Eq}*~R}g*x{(gj$cxK)Y z=B5cVuEiGCDMzxm>WjTi#)j+LOg~zy!Itht*}>Y>GpxV*$kD2@rFGdQ-(I0G!mDYq zS^IiFd6k+#4)5UEJ+gUA=5udu1u7vLo&9z_qs`cBJ7Uhf0aUF5$XAWbn#$yiF6$07 z$95K-mjBCu0l0Mm#XZKN*)va%Evo2LaOh;j{3CZ~&aT+kwiwir#_!Am`~}Gv8B;#( z=lgE1*T*i4ADr^wJ>72%Rh7S9hA5i}G#zbTa$#!eoK8bF7W`N3aKs`I{M7dG^l^0S z$}ra@cTyBgJ}~Kiyz%m?C>4l+=;oiTZ!>U<4plxsrR=kh`d1a`u}$pFwd)NP{gQ}^ z1AHSN1P3>C?aHw@aqct<6bU#)~PD&pMZ4l z14>8-@Ei;V$Sx|rafnoo)BagBg7qwPN67SDKG#<%iSE+c7JwGm5 zRW)fa?>Xn}en10jPQ5$di8t}!z2JZ4ZD)@5`IiWR^rU}xuCw)wh_rih(?f%J(Mfj$ zA0B8?-+o#;xZy%Z9KzOh?(OOxUEU@Y_SsyS*;03H@U+@_4^{4N&DAlsE%z%cP^~4W zatyZ)E)RUEe4D;EH!YnrE&pYM>~9((ml_jN6|7!&jRM9!+p)m0u( zcz)uq^M{^>7~I6uA5)7SCv`e9zhn|~X2O5Y0{r*-rguL;b7oYGzkhdoaa{2(O>!N5 zz?^P7AAC3;8#`wEr)%SH|5Mgg{IvGds_g2un&G#U=UmG$q4hr_y)D;4tS&Myl_OA+{ zPRKJJ-zhr+C1oN_X)#EUrtHOn$61*g0Lfhpv+{_9Ss&`lwdWUisNO&K)*hNzf9LdU z-)zo5wQFK7wi{rM%sSS~I2Vw(1|Si^1pAm!;ErnoAlIsRa%p7+h(Wsi>!CX5gJKV3 zr_RFvub7w^ZcMHFrY+C%+B7wTZd{-9BjBVO+f8zo^xUk%urmfJ=u6kZD$ClWWt){@Gp{3{4dzh#tQVF0>iC6%Vn+#Ba%<1fRwY{8N;pFu``PA%mD;O3v%RklD~G%+D+OH*RAO-p92*C22jP%ua2r~tUvkC zoLez|o#^hO{tvcJviw=z&Y_6A^4RB=o5k94+NXo*wR=*IJoJXpQ7qH+o{UTC@F4Jf zTlHOQew8@)V!K7_FQY5u_!O?6{*dTTcGA4U0u>r8y6DCK!OY1bM|0zUG}pR z#ZuWo*EMKE_Q^NL8?OJ412#jWKw>y@k%}khRakhlvWwZF(=G%4?!#YM{XE2M;kxU2 z^LJyDS`I8XR>hv&7P&85GTo8fW!2*9wL?|)#1atxD*(5q}YKDz59gGWq*L(`(d7amJZdNPPkAM-Myso!EJN5nLPnv!3b3vYI zSEv#i(jSD~!jJ^(`$gOAE#U*cZXVWs)q^Por{6ScP?P5DW(uD+OidqEaBJGHcez_& z&J{1GoXC|~nyNk+CFPcxM_-bD5!wuy-n9xu+0&~J(d+L%&z%3T^Pow0+Ldg9s+O=^ zyGCBA%GKPs*R%crX=R`4O8KHy8@-kpSy`)y`GtFD^!#1m4as}Okq4rjCto}zH9Vah zhk7vKmG%|#fKJl=d~4}f+vY5?S@SAuW+ihl>V8AvjCT!Tw~x*stRGRtQq}^QFzix| z^)Epgu;}&ly}PnVu9M(I0URVGzfUvm-yXcOHTdcvYSQ7fSA?MnDbQh`cfGXSIk2wb z0?Y6c492}MV}a2o2%DbqyTS?tGVJf_P=C3z`VCCm_vF3Y?OQa(TdObqj1B0^6`hJK z0H~`z|9rFTk!*GLhQbMftEaFZS*^B|cG9?>AEvym3hZktu~g7MNsgx{tkAEExg1*` zyk%H_jZ!#IeWq{h@sqYU2*z#&|hq0eFo%w!gtMkej)QO7nQJLn5xMZ&Z=sx)k zSH9ml{3TJ5lJ*LBwhOHKpkR~>oYYBQ-g8vVsT1|qn2EUP8+`vC5vS~c`;^&cI?*ll5YbYY~V;|NjSiSIuyEuQ!)CEMaY#31)8`xioeLir&&8;uW|L4w{ z|9eRyJ47}4nUvSJ!ef? zpJ5?H{_HBVTvko)k2tey=QG#KrNz6tiz_TYGvqm|mb3|2Z6#l|8M>bHYb#K-*Y>Rn zfC^UZKZXaz`BjgzK=jg9ACaNXA5N{)-xJegucc7d3^e@RFQq1Q6z+kyw&D~xKc~I1 z1Gf>jdd0x~DYYvGp6dGFoLvwaHhwfIVa0M-|AaCfDl?&aRWM=Z3C-gV-A8ns_Ml7G zy0dfU+`TNzKZUm@zq=w$>i-#ej!!`rYJz=5?~XzyZ%0|uk))At$J7i@Vh!@uIZ&F& zxMINkv#RQwj$Y_0&fVX3|5*RI(>r&ci(y6bHq8F`LZPkPZrMHV>7oi29P^Q_@-+iyIf`cfNzZxY}FAy#txnCkMJeY@|D4(k(gFe^BiG(2OR z9Jc2Kuaz7H&KU4Ewe3?ePFr5mQ2nK7@0-#^DAYF23-ioLulBC`EI}3qm-guyVm0#2 zgn6LyFfOC75AcJ}q(y;Oxb}n8GyVVk-3$&VjyV_GaK6{hiQxlVYC%SISi66l7glS| zUEKw?Wh`RZ|et7$UYPL|T@=g3fGO6YPJ z;HXi0>BH8))Lw8r1{H+$cswrW)UDM1NgXZV=wHKvzq%g%Yd^avvhJAH97vL#+by3m z|JauKzvkQqC*w@nozKOLg5fSfB-N<{e@AkDX4|i-6NUFD|62L!iZHTlU(u=hR0b1h zIoHw>tcK+4DT#F}ZsdjXo9e9a!(*NIo1;8stKtKof*Kp`K^~rd8*r$Mn1$z&KA=r> zyvQ#l&QFo#9jZu2=Ki?-6Tr1~+9)w)b9GhB={Hk|g7KE|(xYeBJO>5ad{uGFla*O% zn;UPA7Z5X;@yx>PIqU^-Q3*hP%1Y>M6>pufr%S_Nl0W6jFw<@l(1`(RviJ3jDJ*|{ zAZzjP1>t>P#!frcFKuF?HL-JeT92ewaN=@{I7@zi6M4+lp}XUD(w=e_oU~-#uTI~u zcvD?6cXqD(CI9wu4P*8W8IZOE-(K#oxJemesZN~#W6#N)gP%9l^(m=Ksoj4~kc<|p zVqdF1L+RG5wwpJncZ9CL*}c;uX~dNI(b*e7(OA}WNY7vHuk=Jp%Z48O8Od)ZsK4vB z2TETTJ>M=KceQokuodrm96kaHlDA8zzgr&jBcg4|(>0ULkyDn9X=-j=HhfK4r1s|b zC96+^!=VLdz^PEV0{e7NmxmLl53f;uySO5vr@3t0Q$P})HVs5Qn&X=6D^X2YwF$Kt zv-^FQ{7_dAeitC?KvZW>n*D(IM?X3_x6{tF%Apy!@F}(To=rv+>Sx@X*sng%?vDTOlG;L}RcHmXjpkb@= z<(3hf^Y~#A;oYaFlkndP&-|#^3K9`5D0s4W?EjttBFQxG{p=0TJ0IL-J@doce6IZv zscRx1#{v@8R#22fjdFst`Ss3MWy9AZK(276qV`bCB+&nERd2E2e@ycOGW+AwiDSY@ z5t*Cc;j4$9P_9@N$Qkg0R)ZtZ>N@A0P!_KerM2w;@O>l<9#^CY=jL75{HY_(J$DwV z)0HpT1=3RX0{pIb&0QVH+dXz}34W|>S$w2;q z7Q7prORedd_%QFTpmyS*FcHIg7MnW0}q`Omc z2w?yvm2S=&zW?93&YSawcU&{G_g-r~&;5Mvdw)oP7gVzuyNNkWF(c=8=@?s}F{^4< zL=N43?XoMF%3E)ID9zJF4RU}+CMc?dtW5`;W%`|~-a0q}GLMsF|A4acGqW<tIDN3C?%<#7tloy#J@4wZ&VyF+{Ds_M!b^?YkpyAJ}^{9KnIyFDGbPbmr z6Hk7F?;cfA_Z{mcHfp&t?}v|nyMw1{oEN4yo^jM+==M45?b*}aw!H0&6!dy?8rhW2 z!3B2>UVz+e2@FI68mT`@bt? zF5n((cybzyLb9{&=u%hL>=j`2H`m+kl4-bwYd;;vwXQ#I`6ild@73+&_GV%4S*uA` zvuD0E1I_2-Wi+WeP4&FZd;8WphQ0zMZwPol%Vx4230NaYCqW*|!n1^q!+%J@$$0d) z1G0qX0xTA~$lb*M|1){gv_cDeuGI1(Z0AnI)W~!SdSzC11@P^(!x6FXMeQNMaJRFo zv(M4$^^uL%_%1gnM`fx@yxXS+tt0IMGcuJia(03TDWg^B)S8Qu!fm)83il{CTg~ez zDie$AeVT`4{MyrTQNvHB77Efl{THn90p$5`m*Ym$xbaP8zzd~bN&yF}jiFS_>nvJi zn>TFhAmAH?+!`dwVR()EuIPCaGo4|(*R!jUo9K)>4obV> zD6MjexxO-Ajmghg?;2V+S|yiXXNBFva<;qvcswj>`29sF{Ni+CXVW&lfKo_cKiWqs z7v{e|f2y`EVQ5&zDqB={Syy|Ss~6lSG3zvFV>918sTTTFboO%k9X|K-Op@xX^e^6m ze^bQ`?Y#0Y*z^ymUM6qed-z10jrm~c0Nwi=Y`C`i$QK9^MOJ77N$@=&OHGezsEVVF zWhxb&yw!DZR-mOo4PLuFS#VL$lJws}M$wM?ySm(NS4k=BcXhNJM#+&dg ze)QtQTi`Fv)v1y?57~b(Yv-RjgfKNzzw<22`R!Hq@lMroy&Z|&->sr z`+m+khPDDh8#;Sp+k$%SRggZX<$4?d;L6vm@lX%z-rjt} zhvjG4ET&uY7s-sG=uYlP4q82l(dAL?nbIrO%H!-{O@7&Q2D@gtyOx(lScJ!C|6@r} zojLBfy*h-B>D9S2!4O8Fdb&-}jhSMR0+A90$`zM)>enyEj%Bs%m8n85nd$^x{;JTI zH4Oj^c;2`zx{HpUKpYP*f4Aiz>Hm9D6fu38bqokt%54W<&%8EPT`jBQ^Yd2K-v`$T6&wSj#sEa#4G?7x@t2j_UZC>Qz@isxgf{%ZW#!P z*r(aUEU33MC~*U~>s;@60J~8qi3w`oyL?m_ccjKMWV>KGsN~+4brw?vU)2LDnmo=S^Pv35v z-d1h=A*{^`xai-Iegc$g*iO};_6%J2!oLVn_m^zQCTZtlbuwO@0E-#DUp{icKO&^7 zr`ILJ74kZ$RRL1(>IdDtafDLedftk>KMtdzZ1X?Cf_1ac{e)vc{zzi_MG}F7aKG8W z#T?ANPeh_Ol8oxUH=m^agcJL&f%Cl7X9{#0YJ;)GP_{NY}C9^)y8O!e)y_;r2f(9@-5XId#Xo28Haukaqs5z z@&T4$%#>xoQiZwP;onYz1v5*mpU4Jv?62jw%s1bd=NV$=O1`%^fT_3T^^bgH%Z6+~ zF?J7rh0>Tzdle(k1twi-79oQwJDzU3AXREyzq`T;|wLz@5^~Wmq?aK|$v~28X%tnhPrjS`z%S(QIE3G0a>Dr2PIKBWD9P zEJ&7PeRv;6+CB)FSN7s-fZz@TFl89bE3+O^$U^JkZmMj3BsmPHT+~SSU=sYID@tPE zQE{XuZu?+a{gOlPN9j+3L_<`rT|^$imgTe@#o-BHJO69mLib=rL`72=(lGT)dQip# z81FnXo0ZLDUBv9;4SQbw8O0iQPv760*!DwvE_rUX9T?2w25zq$ib2EcuL4?(7(+!` zxui-8*}uWr%pr>S^QJGz3N-4qjQs%+0o-{*wImvEMC=<3O|ud=Gv~Rgd}kIlceeS* z;}su?B(b=Ilkd-ifv|J>9M~Y|e>(=_X6WQ3U%KOoHC(2-;Fl0S*dX8p%B#PPQTfNF zn0B1@_I=`CUAm_J)}n8In?0AheiKB0T3nb})smXR&Gagcwq7|ZKW#p|1eOS^ropy& z_L^my3gss-9=ZgaHZAMp#^ptbvjOpk;BZWiKtFEanr)h8$0L(QoCNYs!uO=63r}|N zmm0nyVoTvsrx%0G=~XQY$k*5f+F)2n9=8SQ2MvxLRp$Ct9XVeGZ|N`&NRtPZj~WAl3)uIGF8 zuQz@9_U2SD&{^--%Ewf!;I)6fG+f*IwtT85wIwkETanX(b{1e*Wwh-WGKkHb43GH8 z@_VJltUd5SrR`Ntct@Tl{;4d65uQ4+yF$F8C~epd;Gs$xe|T(9C+$d=#~-!6bW87w zD4SJmK|DmkZ0}}1o3}{%yxwOXN^Rr+`roJ`^Lkp-2H$;%sU2nI3LzaE6kV?f4ZGJ! z2<^gSA6@rRj|my>XFz|k@u)Mvvcp*;S}E+~gW)Fcu8*e^((Q8LLfx!^x#Dsi7am#R z>x@^R5M73XE{6tM>+=&dl1|Hu=MD7`@vgWMBQmzy#)6-cE)4Z#KRTpt&T<9tAiJZ0 z^|JP-t|s%N!{+@7Sa{zt_L(?fjYCXIBz&Y!TZ>t66I36uNuu(6Z5uW2M0D0WBd*ZS zx%WiIK$TQMEk?{^%j${WUztnoF8Rb|EXxX7CAzVMxP^uilCppnMA9^*A%*T8P?VBZ zi0!t(-lxQWTB!>CLlUS;$TqiW7aan>NWw{BBMBa8<%0Wu*f+|_NKod5eT89;`}{54 z$Yk&*KK#}4>sDaKh;N>bL36O1C!L^`%5J32SfQm(6Xf_QY#YB+K35} zvJy#h0@X*Q^^US5lr0?oks6MDKs#?>!fauIZL&JazWsz$oH1NNCM$(0_v;qm;{~w0 z(MX-2$fLzi$R4h+f^k8V))roH*iCNAE{3*h2kop$)&fPR{IP|AsrV#4}8zhUtg6vw!)!b)#Xq;F%T3vt*Cl{wc~v9Ja4=| zo@qh60Zc-^3AA5>fj95X{@HrtXr3^2BSL5B5F3Z#X3ZS&uwwwik-U+#NjHk(=LT7C z`5ur_dEqg>R9-zID#%npsw5To-on8-f1ftXu!ZE%lITf-B{xe?@#y<2FH7xH@0+J% zY?0w@NRtpp!&uvDL`>NV7~8L(ht?`;hrADnmb@rpXrOei(b67v5Q}KT0Xt*agu2X2 z)#M%IH1cs7r@d&v`8v6(4=sdBda$_bAIutVJ|4$_YDZO#a65!07M1@<_?u5AC39tK z?6&;`&TzmNdWak-T*Vr@vHkIB8`ZdsMVMd>21RP359@!uNml;+YC+{8ikkDk)}B*} zvaLMjcy=v>e((yzvUo#_NZGsFrcnYRg-2+QXbduO;Zk6`1J8YcA5kb|&lAydP@ezx-gu4CwQ3D|G$ZqQm)7e# zhB~AL>>-0PIlFEcLFXYFIRiK|DZtNGzyFih@eq_-WpZzLgoqB;k)*Il@$)n$0Svrg% zvB*19In+aJr$Wx*+OD%UQd%M2kcZkAZ2sG&*_Uqk2J?9YoGqRl?C&)s7ls&oWDXPW zvB-|(9DSfEu=5ZZljiFhPo z@;IJuv;{S^O0(2%b{%2STes~3_prt{ARZtu|4I^eyg4Yd14`?f04rCl@<=iQAyh({ z0k(YlFL88gPmNgKCR_@yOLv%~mx)L&D@8&YG$haS7+?SktJMd@_JBXDXp1-zv(6K^ zc&P5w;Mh`}P~25?{@+@TEvs5S1zKFxAC!buPt`i;J?Cre1vmL+y>O=SelQ_Iz2v8E z*QWB->jM}}(jD>jG=90!&IWQ3?TXZA0=-i3ULNp+8JD1Z_{LD*GMaz}m*EWw5+``U z|0BhQ_-A5A5Q;nd)hl3sz8tz}B=*zwRmsy=lk8voU56YVSgT1ahSd?LCVhm&bAZC@ zX!wH^^J`^JH&w%Ojp7AUPP{s`7)(m81i$+~NZplV(oLFrk7{puqE7xe zc+9OL6t0c6Ar}+Rz3WKo#-`;0>8;^di@yqnp}tQ@gDZ-lx%NeAe+SF|{FH*Mwdi*U#ZtDS3HhDdOXzny6SFcL~ zTt>E#LX&mTV{AGyNV+@TurGJtecM3>5^%+Z@`ipmNL9KzRj}!gHtpAxH)rG40D&IA z*L2Kaz%kC`T|xMndi77Ct7~!#pyEAnSzu8SFW09deP%xyaM92!p}a3hB4;hoz{()( zGkX|9Ku1JHMDev-0>Ew->m8k_PaTwbYh|V+YfS>My5^!?w>E^*=37{izv5xwx24mN zY~3qI9^Z9|Homilwa;uf_Z%8}k>f7|~2Koeoi!?!8taWKl;? zKueof6hvdqyfZng{E1<^mq25zI}{Fs#EL31)Ke`jUQBRpBp=DB;M8=7#2M-~iGw~o zJ&UM=Yi%SEs?hIzEJdI5L@;-H5R)gzZO!x&i0mo;r;9T!W{@UAqrLm|gE8|OQrA-j z5&g%gp1o}ihCA^KwpW9NA4|}sB_nln9RIs1{9hvW1n+(hYQ){o`Qa!MZT2MyopO^2 zEzMwn5~a$)tmo=-`F3doldC2J7_9`^FgJe#qWSyS#*vQ#EgR}94Jahw>_Uc}N8@9k z_0xJNr{-2cF-$!I{&C&adM_rJ*3`sH7o2D}h4NFKHd8*zkffI}keJJE3mkvsv*7@LHBJW>S;!!};ltH~)Z&{JX{*z@y)#?vCl zu@9Hjp!1oTt6y=dJk9eDw1F>3wz+E zHY2}I@=dK}@8?<_(P*ighU>C?ULs-89uKpYlv>;`@8M`zLT(cVPWD=85h37dT>urq zl_PTaBsXP&U5=K$+By$LtlQfv?U-#*P64>k-(^ z+cLf1u}xmFcKe8I33BUH=?sY1Yt?1%g?4y4yu8YpW^>~2H!TZ=BX~GMAFd(yjzM#1 zqTom2vz}r@EPS73@<0>O8De@ui>A{h-*op85vDKCYktN-q|nl~DB%hCR~Tx#MLWXDuD=Wnhr(?Rl&zAh)n&+s-f zT_f|omP@GWYhZSYAEK8!Te+|51nN6GGN+fD6S_mCAUj{5Xw&qK;dogS?K&seg(*_{G*bWrNfC0*Cz|)q!N4<$+5$K?7vf; zK&QN4Su`WBOFF?r)g<;x7V9(pi^bE8E*H25F^_U`bS!l(o;)ZAy$g2~37sRCL{NPiDt@ody%N5%HV-B-hzlJ?s?tOP!Q ziEEnP#!mc8H^b|6+AVWFhvpi>pjl3nAG1j2(%$3iF5~+)kelN`Aq343ZI7y^S({S7 z7ntTqhrC`r=*00#feSm}pxpp_RSoP#I088il>g2U)PN0%+d9UdJ?&ggPI-I%RuLk&T7AiIgTP;ZG zL68%LF7=2PyY61ujJ%ulHk z8$KqGo&^iojb&&5IyADIDgW|ks(2Ddwz3FslP|Q$dDC!zEn2D$^JV5UEgZ=Pam>x( zbn<*36OX&2;E+Ns%9Iqj_vUs)OrHRvhi~TvNB`6Jgtwp}56f@<0@zQ6I=wH{9#dNV z1S*E?hG=hZWu5vhe}MG@!h>v5kL(Qp`;(ai*8qfuwsb`Oj(9iHWl8a&SW6n(i1k`x8dN2Pg)m_^6 zA89O*HG$r;SN)3n9l`I;r&V3cjG8Yk-M>p96-=y(U)32(P!HIvGD73NY&(zj?#>VC zozO@6CsN7CNGSWKLtxg@ay==`p1Asv?Nyf6zeoeFC~vgmT?b*VmNN@gr=YN}pD=&L zVSTN5zw(ZF^pLLL0R?}bqG#7dAUQYVX8LXx#SCe8V#Lq0dj|HRq{AhZ zwTgu5N!D+K$SQf_eb2tw@66avDonQ1O!q~!V5^I)@SkYKxaEyote0+{1VQB|6J!Sc zC<(sg$~AV-#h_wUZXi>7s0I=l{e#6g*9U-6AUo_>Ku}yaIC~;XFMRA_h%KSVLqHh! zx^AS}evDh#TWm74AV9H&A3aMa)VF_zx^bRn=;dEcy*0VS8=Ts>!&WE7~zOf=QlN*uWZIH3ZT_ zs{B5isVmi{IA5UhK6W3g#)70}kTpmS38)0W&~i=mas4Z?nc?oV(h8Wjo?XRlU2asY zPmsDFL9>x_IMbL|#=^Ky<-xW?O7bC%ss26n&rQKkaK+d(;4gb?_)G##p-guuy)f<~ zBEoda$9_nn$VJ!iv4ZF7q4vhVXPjRc zk&WSC+P?~cyO|}nLOMR#gUNYP!+=a`-D8qK?mt1JMyS+Np@fcJWx{r6U1_6;u^oph zd8O>M1%6N8_1edqbA*M^m-MSKTv@Y?veqrQe5wyiqG(Z4*$^@f^k}nQ8Zov}g!(h0 zk*!3c=cv58ee287<@!~Z8SScGVxxxo)f z#h#m=ID7BLpQ@W(M9p1wMO95QKkP(@osYKP z-T|sic@!#j)lO`9IcZPnqb|? z8(JM}cspN_O=_-{=wAk%3bNLwl)taS5Mp&+M{-w%Q}9L2d+Ftw?}^BSPkGt44fzy6>_XLBA;QYG<*jp5ZvgYTm;iYK?E_7wGp%!H{h8HI>Kk z!Qy{wX8GULfrsngs24)41?O>w_1Daj_2#HmKxV~beVhkdH%SoH3qgYkHU#EzgS`bKI3=e!<)e@#9rEVU-QJes`F~%v7~!e##^W{*w)QFfZWDg?g&r0) zZ_Dv>@Z&yb(~bq`MPk1>Vh=i`Y`5NCgS>IqT}bO2ZS*W>wQWd4-p`2~tUP3O$*);f zA(#l)nGIa$f#20^2ajY&R8{7@M%wK|q|qi!s)cuZi%SL{(|owA(mX&Fl1rCp+RW`2 zj@mrHT9UE=ZQ^MCc>FCw&I~*uNwdTZ;&_AENY1@rtO<-Ao%39Q$~b{k87anboIv~y zI;X)9@{)^(-4e_!ybKrPi`f$0X#zP9H7jRjz<;Q$hT<7=Le)OSz# zr}C$|9M;RRn8jqGvjj|KT;}S8&`9xYW*PLWGV|ZnOGSpk?2_UC!h6QFG1hZ~Im<=VA%T_kf z^Q)cCMb0y~KN8SD|DvD-$*O+cg~d8Qj%>9B5w3Ludbx@NYN(sC|N1@rr+79Tn1=UZ zOvvMz=VwGj>nC7)jFRVt0{T0xQ{%_3>2*N2~w!77pS^`1IkuwF92j^k41vns*Q!vGKq{f1q?T!cAI|9Yf$ALm+f>k;m5Va&F%8}3QUZqy?lI<8+~8VR_M zb&Jb6-VDe2NS*;bVxfZ%;&87V1Rfc++fDh3ZOxPa_bYXKG{(}9=${jC3~c~X!x*xt z_7yQr{xs%$pxCD^faJA)Bw4e#V_Wfqz80PO1hu`Fg3`)utd!1}-3@4CySY zL`(Li-;LWa`SoN~oOOK4nzPhZ()JbXCY&jg`-`YY)oqf)_q=9K?j;8C zVOE3tZKL%4GSE|iNX9bCCq?CjdnEl$hWiLl>&->U&(aLfd3%q$o7aMvHhm$o@&)N? zIe0oxiZAlxZ`cY`sMQk{D#E~n4+D{52?TbLbChQ45oq?&+x@RtJ>Ws4m@fDth^Gvd zFr)47$?1lvm{JM?^~w?#2aeCMSTJ9J(jZN?vhMZ1e@hm(xvUMIf6k7>DPqaf{UT2B zD`?bDp0FkpAK&0nx@eDL!d%_~hngncHf|RDn0Q!Z_dORH%4+-duB5GT2a%_S+z68H zdtss(n$2#<*pb;=UhyhZ7ZHz;ki)^}|B?5lT`hr>4DE|(Ax(Q`)ud@3PAbFU#yZX zzBhR9uNmMr;t-A$WH3ZD>&7kl+SloNARbn9{ootQn2}IJqh=3oXrwTsHzcISzZYYq z#1opxJfF^t@C1lP^#{s47w<8dANgEf?zpaWYGMS-6_tvy@`cJ!eS-O0FrM3yWZcS8 zbweVO!Z91H;AEcnn6U%4gU8s-zO%>wju@QGG}0%oQI)O|N*)o;|`S28_g0bS~G)_uR{GL-befa_~OoJifC zM>lJn5n)S)vf!7vjeS-3en@It2lMSu?rPv*`seq$->{7wjd>xPF)7VWn6Du@pJF}zAmw7V#ccb;b3!psZei%yr<>SG zSnG&II!7|wvb8CXQ`8Q8cE7hh+D37zXu2vbf9ya(w)$C_{`A{Uh>`5QEmxO78V?33 zPE2>AgrN)(z8H8TDleqot;HtL@%z^CXEjs-9N6nCk3 zA{#XFh_$j|^!fP5!JAEyf$B$2eqMRBGH7puF!RoUicsG{PoxtRWf!zKif|t>E~iy9 zB1Al1ZDbDn`^jJC9L%NEL4?R*6-Gpp%??c%$J?u41nTs9 z`Cl(Ef8Y`kp>3d$p7{}y99I$*8**@&JR0P;Z6!Vx+gU;tY)rSC{sQ+`?0x+)z1+A3 z;OUQ6h|<#qLRZkAp5>d9dilPx+rWP_Mm723^75!4i_vL=t~?=2U^9O+o%ehf#-(}s zw;v^`mc)WoAxKlrEq%I^QKd->jaqA-CGWEf`mge8wlFhlg=%6O*}i5LG_rU)-HZKn zh9;fI;!NB`L1xk_!ACZeXk@&be&0j7^0ZbZX&wa;eq-K4xshPnW~CU(l<+m6+a^}K z2=mc+_7CQ-p0HaEJ@ES%yKZKY^Jcu2_zcOIbC=MUm+e z-BF|0?dT`Dd3_wt*uTi!wp(0+(1z%`WZT-V2gGzoa7yv=uyp8QdPB}`2z#|}C^b|8&l z=xPrPrNLw|X(Tj(RP1edG919A!S1gDMUtH$39;;DU3E&@_*)kK`Xfq&YI4*kwX8S7TF_dI@}P&JP|0Suu3URLSWZo*)cd;0#LE)FmQL?&~y>rJ=M=OJP;OKBE~b z_Y(}6Ui3x}I1e75mk10y6OUqMfys=c8wXEA==zcZAtW!vw(#AuVsjzLg|*H)>lmbU z+ivV^u=!a7nRsXV_MtpfAqe3RAH=R{Gzk+vwasYUYc?v{m?q;rw+&)W0}{)=@Agu9 zj>NkEC18~2JK17QjNQ%ugnFZ6=lTtE$eLh9QxXylvtTZAntk}hfG;Rn zTa8+__#X?vL@8KTgwV-rK_iv+8Bis*&n+1yyEXHEdmx5A>}&wYP#glyK$XOf&1r^5 zH$GriB>79tz;C{rTiQ&S(1#?v%Gd7ne3*JbkEGp;x=V5Un2$F?aZej*=C?P8Up! z+y*al=L*9#tOeb+rTF5e;TQ_B(eCehino{C@(*C^5`)*cki6HU{fAy4H9-)BAe66t z>~cQ?WuY2ar)h@mPt!-sRi^-wTdN2S@{0@UYNYVM=4QI-*YcxTv=!cdDu4fz!q{#j zxAYr-FN~2WX4&AKvvSRDpJ<}I1TXP@FaG{@6D-c&<{kqmh-e6ZAT->(Dw+Pcbm#t@ z(2riCk4-o%rx_G=Sw`&+bhIzMpT2qqHJ)ISZ%fPzyUmS}UnbX9Qwwfuq#V9}PTvLm zr7JkWPxmta^$UK8LYVEu9XLtWmos)86L8jU2o~EFg`B|~>Bj}0f_yNA)t*MTqPC&I zpAW+2XOGuo?`x-wmbpytlmm)&uDL}I@$goIPQL22l95@b|5Vzj^xyh|MvRNbRGm?~ zp5*+;qH6>g_6ewNov*|h-fXn(U(_}gSo|P&JBMS$E*MGHgL63fjr$52iS|tWve46$ zeh%L?5PtA2mx#Em^ate`jZ6_G8|~0$9Vtk78;>)temy?Ya|~0LbstP?kgVS^UAh(b z1=qIxVXU{=QeVI`HmqN(*TLsyDbq878-hxekgV9U4>gw1E=wZ4$ZR@x2Z8NhO{w9D zXi2^YCMW2tV~y&jrvvg zwG(LPFf`fp?;QqiW>Cg(+hx$@`xpe%$~bwq z{PH!m56Kurd$l?whk*~8X+flosi1jBD=;Zf;$z;tMm&6j!g8w9b;<_@YV-Zv)y-)! zF#bjc10CS65R-2gS$~jHi;ao%<=iFq z_+%d~w-EjLf1i^(s_BS>`k#W-q0*8AbP9=XAu0I4`8VWPp4iZ+kp7*QNVL$%EHVM> z5bV>2flb{H4ZC13x2NxcALwWn>#h#sMmU36Z!du!lI?SLU59jQ*{D{_Q(LB4{#`hk z4E@_o!!y_VkzWO^06VhdkD|=6fAQe>cR|Xf|C5e5HlBP-YPr)eUiu~YDJZbE`Xqk- zp7lgo6D!iI&Ha(o%he);Vc6^E^Gd6Z4L8>{&sw4LN)w{mRrxdr_m|hjILiTpG0I-a za3PA4cXdzkyypY6m6~h$aqnO)lu-EQcF^?30!h`XzV_azc-W=bS4hb8q3DlY2oYL? zOm?0rwo=4HDz{w>X7|engY$75s6KT(gOc$|%X?>jY6q#l0w%d2yiHeNBO?LcgI0=K z0t6Y1pHD-G$K$chf(Z>gRKYO6J%{Kh(9)Y7~@xo(pWFol-) zSgaCG>;B(O9zj@G}sFg!|$z9#!GYT{X%M+cUIUY6@W?0K4lQW_zs5u>%G-aD-&2B-@6 zbdcS_Iq3;$*GY=y=Srf9>mN(dtqQ+>Gs(3GclD{AduUrhF-UAru-PzS>bdzTMFWvt z!h1vFSCU0g&NzxB2qf7|yol>mt^G6r2O{;`iRx1ctUPXks${#B@Gv%x{;|GD)G#o% z;=LR!83y;h!bkpj%W4GM5TLoW)AhjR5PbjsW3@fB|I2R`&GoT?$mzr*y7A;GB4Sq| z&~cof8acl&8Be_LB4Xz~`JZrRI3)3s!@v7(4sb>)+Y;!ezDZ2FF531nYN45P#$0gk z6o!?t;AM26gVmxn!#(U1w?8wAPLqHW<5w-7?ENv$CsOyt&&q}+7pg{%Z0xFHPN*(Pgi?A zHqL&kNK24_lV$vMZd@&rM}giJpymIHtueHMXJ=P-Q?n_tv0W=mI-NdPX<1vmG?>OV zI-3J&PP=JJH$LrbPG#=9e2`Xvw2kI9(ed6OLj|WzCi-}|9zR9()EBGg+$8ZbgcXgF8-Iic4lI}gTxSU!oa&S3mk=OfNcDFk&g{b8R_WT}?B zUm;i>yO1rOP@W*5mbU*yI%qz5l(r32V75i6uSArV|Nm}6SSD4rh?OL8t_iN-@5^r0 z0YJFGZdICDwgj2`&igdfO6|EZTU7S;*a(KbjkS0U!!jl{tN+0^ICJT-mDl{@oQLKP zmdFELrUcZgGqY5pTi{n!$ZUgOm~l9t<1Ul^stGBzWn^{UN`gR#4RFiYZMdY(Lhe=B z?9G3KC08htE|JC9)dbckG?Bi$9V>7gP@~SFl!Mc<$m$XLsffyK{S~ncnvLZRrK$s= zu>(ic30(S0bcw%{n;}^|CPLIHE7HdHs$rb=-hJQM?FZbjopN?iq`q^{VDa3?d@c6~ zMav3)Bm&~C`8;>@4uNxJ{>Nw@G>>}EJ^;}?DE4XtqcXP)^di$Z!f!DeSg5Q^;{;z~ zL3}Y$`T@kld)eV7wCWpC8js+>tn z{D%1T7LAr+KC?qQ(Ntsd8XN6HxO9KOmacrdz0}!4qs^IRc&gcXJ@Bi~AA)AD4`vv; z7Q^BU@2Y`*@ZG2QL}M@Z)w8HOQJU4GN$?Qma1Eh#pWx{|XIPr@IA>_ovy+z~u8()0r8s>MAvGh3#kZZf zeRaK4-iao7r^$P5Ig;1~RQre*06X}lFZM#oU^QiHB5Y`&5 z-f_x&-^j`OnwzX}3HaX9KcM^wW}p1d)kXX(=#nbF6|%NNG>w9SbIOBCdDCO@-u2AV zD*An@VREZT6WZ>mn}G3XU33CiZ_o0cxo?it(*HCyOa2|PMH5aGN0JwGHLU0{9M?_t zVRRk5Qg?)kt$!^f^v&!t{QQCt;q$)l6gdQV1v>@=h5Jv-=)0x|%xU>bN~=_I-Xpfe z#z*J`lOo|Nd?f>1$ zMkB-^17nMsaQTK{7<>TbMJb<=bF`gbP@gmDFvrX}eUeD9eL(vVlSK?93qFvNo1(Yb zA+#l$m}4Tyr{gWS4=^5NI&#uf!)3dGyN6Wr8KyW0K23rBipFjy`hb;%o_gJANgQ_{ zn~07@7(Ht7=QkW8cWfGZ`FZmUyY1z-Yj{8#-L?w*67{M*FH z*o|>}z|r1Im!CW1~hJE$6drLdXPUlItc%Jf{E z{cmFA_wCGBb?KK8myI!!iEu^fQ2{U4-7SA6|GBqHQ0~cU%1|4FgEM>h!%ILYN%FJgscPX?T-ylL#GY*mUWSc8%>XS6LzH4Hb6 zIa9A^jQN{6!#!#nCPu)k->lqQpdhQV02pUJ3=l03%}DeP_+n(KmrH?)i9<(*$a~yD zJV_?(TEi~I&Q~M1*FkRT&S0!phcg4{iWJA0lwLXf@{^GkR)arHp zbzPk9Qf273aeFFOd*If^Wu7LUU(0q@mxq8lbTR7j@!#KRtWA2MG9z(q{M89QM9sp^@r+`D#KeRHLReBxIiiDC9yK;!ewQsPpziya? z{n0gG7zh9+p@$b93mnG=(wct@JL&ndR%!SbgVKG?5Mx4CeS{_G>n=kC|MW?~`zG2r zLNBtnjiO@2HbB34aI@(ZvoYgk9@kll1-BYbi9$Q5J`RJ@di*$1i!e01BXFX(qJM|T}Y%PXhsLjRM+ z<(ivGLp{e-zgqEygZ33qP&aafM1riNF(-?TGXbpZco69A28cvr6s(*#ARQ<5borfB z>`6`R8=_gzXAFZ>-PDL%56NJGttEBx#4TruyXmWf6Nu~E03=fNMngBJjLmTO6S?{8 zul|aYb7hA0HBA#IB+_O#SLaEy2SN6gNoHusSL$Iy;~CX?&tH?3fOO7&=Oh7qUh7o` zOp$dV!M)X7S{voF(FlDv@CKz*n8$*9ZrVD|c@|qOtL_7C z94lI5_AnT;a~Ct#ejz0kJ{psAM(lt$8Zm@S-@c1?J+aJaEXHqCp^Qc@57(7>q^*_l zm3D0xtS+YU=W{45`u3#0H5D1>Lzr?jPX}o`C)AG2&raDUFW!#W#W;Ko&+JNWwqVM+ z9~}$^vsB#Z3Zw4+e*)FgyGi9?p8S2VUZB+$-67p?Q~)L3HqP&I&GZotq(0CvoYP@aEueRO@!5` zWc&it%iiDQ4#N?0;M#U`=pWqIaZ493Sf94IyK=lz6GlWe_YSS-1Q=g&YxNYyXZWX> z+OtTYmU7uY{2TuKDcQ@h2X~aIYP8LI)E(W~qhJASogd#&pjAUmpBC%IeTF~rQcKjy zq7_5KYy@smrWhD5SXN7fY``7yo_+aF zM~L|WtmhF2r+)%W*O^IHL~X-1U_dr~Pzr)j`UT6&5|NST3_&e^qB?Gdl8&*{N_Vg% zE(MboC8XYdLqZ*w#<^bnziY=aI+t`>bl2^Aj~YBH#7e;b0DQ?y4o;Py=pMx-y&+uL zL4~c`raRXu7m_&F76BGu7}bl%efXir&*l;ELx`_ekN%s}&lVvexch2kVy~XLYEGGe z$o#wKwQ^Gn@A|Hfa%fZB%*LXRY6C2|T{s8IBmld<<)zP@h@&AHV0Z9j>IbSGz2-=a zdLL)VP_GEyx3k=(rATwf0qA9cdc5GnN>2NtU2as)1NCk2BGUg{2CRsGL}AF~`|nwf zE;H)Ss5T%PScP*G(|reVUR#+?kB9g2lTE5R&R=Lq{LBDQzBNBDwTcnA6N>cp$T)8; z^w=DvV3KMzHd&}qKR&Hgz;W(!?N^JSYXX>?vj)_}NHPoXlTjD5*)l<*%Xk$0%!l4^ zg(yStavh^C*6#s#%V&;Rw88hk=NYFUN8t)0_*?|g# zU?5pk z466Q%;b%i9#eos4gH@f&_(Lh5X*glL3D=;U`2xIy5ZjrLtT7sY_do0AKcdGs{&6us z@)!_-?-eD(8!G!gIZ6_ZdvYJemBo-pw_jXubuY$3Gk`SklaESZyNA>%);d^WbbRIh zCfq3l(i2Ex-9nj5{7~7AV5Ui;* zRRd?)_i=ojxKPu~bA_d`NwWLNo|J5u(f&D!rrMAaPS7bTSu4o(N&OdZEf<@=eg)TmE}C~E1-QT@hvEI z73;cDx3=n70kBV=X^DpiVXQ4osa&eor2ehKON2gj0=I{wFjSU#uKjR}2kz zOL4u)!J81{C4;`8uwX$uT*RXt3$2)~*Q~DzE~kw@|FFt^C9ql<$ePgu(}i{s{J;Il zC%qfE$Dp(GcBv&)g-Kmmh@~)M(Eb}O#*m{(zFws zXVJIz2wqqE4-C=E=f5?zFv66+?YjSAg8u|m%9GW`?3^BTAbDKPn+3xE;5LBVEaDT5 zk;?`5Ay^={%0`YWM=o4%%g zv5=;7B5~GVX;d4s6%q~>htIDI45n>J>i;f^)!tLOYsB3u-E~(J)~7x4*$~U>~u7=FYdc=O-t$V$DmN{xlY3S8%MPY!XYnU{v4(guV$ArJ>W z{T#4U)eee+W)#MFnqEO!@0;^8!CM+2JX}+n+X82=wDWCDvSBvf2;p-o z;8iy=n=Fl5mmrY{#ux&hil|#PEtv13DYyHT)XBdGz&Opd%P}Gt=4TNX%^JK-h0lI7 z5guvA`%J7n@a#_%RhkAdM*}&(7dK1KcNZHKh;xJ#hF$vANR;Q%i&QTL&!2&Y+2=H8 zZ1OsYZ*@~fPBf_0Xx0MP#F`k+~Rcps$a!WBkM%1BFAm#iSL!vGC5Q zgQup3uu{I5y`ABC4@>3C-69(Ypd`x@QR1N>|2Nh|XB zjg0tWNan35!=WYl0{VisGG{?yyQ%4%a{254;j=bfgX|m5HgBtSStTZ%9X?Qp zO=fX20_N2bk|X}p4YchwmDiD37b)WjzC5QnP+f>MV!>%Wrnduz>C4&9YR?R1Fr(Bo z5o|saafd1I*xz*{0YqJ8RIy<@*BUu?9PqY|6xc-rfXg7o)76gmH=#!+( zDDeU4FNm_Sr}wY9*zf;VIW*bv`wj1J3-rj!NIpN2T?y=+5hi*_@@4 zOtomWzNYWVn)+!*=Oj>AJ!ip^J}vYuT$5%QYqw%HD+b6|nl|Pam?>4Y$%tQp=n-#! zE;bsO#Yyh}1bO()Ws#v`-DjUUrJm6ml=Eo&wW%wo8=Hg&TD+y<2*>rNmY%wLMXRL& zDz8ak29$3y-rEQjzpy)@gY!5_lLbp0S!+T}*8E^C@U`URIjcc&LXl3&mqw3A4~9+71U&{aU}$)FHc3}?IGmKzF%T{wOHDgRiBT$)}c4|kf{ zGhcD<&Dk_E{09iBHg8QytFkCTQRZ=cvdwu5$^PAzF`N!-tNL8a)!91+4LbkW+=6kN z_n$ylf;Hs4IaLnCkpfZ@w+9XLbHE@HEjYF@`T>kAs=?l>W}?mI4Ksxn+Lc1)$vgr_ z|7AVFI=tPBRl+F`MsC>0Xp^U7HA)b~r=|9|7FwJsiviAs5pjY7GL0G-fl|>sYl+d# z!Ba)_95!eG+(_MHUR~!y}Aw&JO|{`xkTw60jFL61uJlXO4YK>keBkm5K>S;fScS&))(G^O3)+>$ogMuFMIQ#QKU zY-W^Z*Ht;r4kQq$BQJJYn&BH26h}pHOd4V8FXCU_KbJd#JTCDQp=kJ=LMBC+P!=_* zd>w^4@*ggMzd!9k;V8a1VV;}@*UkcsAkz`kn4_g`w>GYQ4;nO7y@!1e@tEb(K1jLL zO$Ulr&FLh4`Y2wkX~RT@(+$zeG4Z~YYA>|v6MHu$K(*Y|=t>FRm-*J6uYvG~2KNYB z+z#D$vUD+`yhd8KPhC%HlGF%HZJn8Ya!bjTaXdnYe%erg3Bq=1i7MfXMv$d4`K%&% zQEP)fkh#J3_LI!Z^fHxwm{jo_=4v}fLf=g!v~Lf93yZ!17i z{mYaXPuI=I2j*ZjhpWC`>B8FcmwmwvJb1;oHsIfGy=p+5uX+{ z=UUI0yI`IaRUA3^rtVp~e>Iqet4~Yq`PSb>GrhgIc66>BQ@uiuly!nmsFB@u)Cse^uPix`eL z`M^05%uDQf_mu@YtrUBKfEcR6Vc~{W5kU$uDMF0&VXsGL#>#33-zZ=0;M%6Cjn4?G zIRE@+%wFmI%`F8M3p~cv;OEn(k@p$&YCA0g$!$?Eu9N31!{WBUWa8v#-j%tbxvwdI z5giapU2wKY!-!_geJJq9Ie0?^i8*evQs-|7q4r_ga zr=v76$Vq8n@6ZoC^q=RyUG!g`pRKOt>^R^0Z0l`|xt_e?xg}lX^)$&xr$D_a66PoG z03?@svahn1MLA1WsMG|l`CJ3{$?;?~BDtzh8(f0;d&m!Bvx8N3B)&otg z&KJuKUM=~z7052>x+|Zf_SB0yx5(fxBi>9<$Q^?C98NSluw~`0IDjkWXK7_iXI2ciw#Y5>Ej8fzkLQ!~%)j zHJSpa`40-h!oqs!3WbJ;^PS8lKGl{~?tkGUO(Vg&upe2(~Lrq3ml>)_zoINa6Hw2Z`Z zeNhcwe(dn5xQ_hTa-(EjXM8lRp-gtD;lqaYK}vTPM-;aAwVqJ?l+N_sVj57JqDsCd zhCop4BctCB8=RN|WACd+vMeX{G_W%3N zXM$H6jDXZY)D8E0)J9T|2Y#SU%CcBg73hI zjTMAjbK2)bK6WoYc^WTJjHf=6xHSIwKMQ`~--6S@U3rni9zH36sU2dkZEP)cYMdFF zDf!Ygvhac4nguCTJb$0>TMCVZ=D(vCbWcL`e_lR5E}9yQ&AMd4QBO-1>^Fx=W#~-- zgbp*gz*Vhl4GsqUxGtIFNr<8>0T#!k{?7#3r^DJ>Q`lRVVXET!)OvRvq<8)RYl#_jk1)EEkV; znSPSXray?&Ee{$iy1NmN@xu#bCanWT%jeyzPpc>__b_|d?b%-DWt5AQYJOXEMnh7e z;7ig21c)AO4^H18vgHo;8KenOIJ^Q|PQ4I}OuYswy5lCYJ;=NpMi42gOGWq*RcRU( zS_z~)6!IY|t0o4yKhGuq;^w!iF4DSa@{x+!3R)JSJA1FUj?E~T>!DFeKanACpIH71 zb`867;^U-D&B+GR^FqSKkY=odbF0XUv+d3l!1ER$Q|2*JfSBb z7^@Kk@$(nV;v()&P{su4_K>!Puh{i>PhP>cm_3o@JnvQKjIOu@Ih2jWg_BJ?+?jYM ziMt&>g|g!z;8QbQb=^efG}U`elFE1xbYFR zYxeUt#@V+qw~EU?n0n7Ym&jL-C)C6`vAMG(nwAfYvJBVuOMHhJ3fs+LsK3&PTs~H_ zNst0AJiJ8vUl#+J?d<H2HNCaP0@-ap^*PAJsi(qaq^-zocIk8LJU1N(TaGgxsqoO(-tsDg`uBAD#=tJnK_ z;9*?_M5GB&@uRriGOS#Lr$$$zEbxD}SFUqA;~P|w9Jy^*k@E{lUKwu$48<(xHH>bF zVKs*uxp+{bU2P#E%02ooPD8S8HGKckZOCU*D zQK6kQ0W}RbV#jp4S4Vy_H_e_RF@;7LIk6K>7oh5o6n~E%Ft1S#CtY>@M(8E!njylv z^!@gsWgp|W4lli>u{~6$djypopGe^48ag4*;|nup!$7_{b;-4<6^SI;J&3=7Fez8g z2gOcPz_6Qc_~Bp1YRn;y9JkWpwt>A#?_9;|47Zt;+7{KPqMFug^>Bo6bco2Z0u z+WgMoQqnTeBN;bZ#!yh%B%-~~khU1@!U;u}9~`@ZEr*3#IAR?`H||p&!lzG#vKFYl z&|Fb8+b*Ovb`kxe011Tr6Czv3#%xYwXh0)_QUk+rW*=i}+QB%XlVcHOBF-`{IZNx; z=GLAY;^E}lY=+kR9o2y_^9SE=X*Xx)tBG@QM@(g_3 zTIgSn>JFfFkfDaqS5%KH(oEMMKZhT5Z)~gp^OA77(TsjZS;?xVWoF8Q| zylaQkj7qS3A+JM569aP-?t=DNLWp>J^2e&^*WIr8MsvE=!6kz5lq&}$d44B+`!J;? zLYwYB81o{C^`TcA37g}@fb)DC53Za2V8L+G9H$J-NA~=79)ZYs?cAqhe!)HhI}}lK&VN)k;{AnZ;}4? zg@lrvDdEuw@%jk}pBwv5xcOV8OKCb#t^BJnaTMdA zYBXKN`)GaZr1SC0!X4#EFPC47mpL`u)M{);6vRcX*WSJZmfsI1Wl)AfEoRatx{{8x zM>nnUfnFC($8v>QY8%cya6kR0ylaOQ(m-c>@t$(B^BOfIQw*ax6+{A(&MjCTNb@gP zN}_U4Y^FXx+mb-)xj2BR9Gm(SX@6Olm%6Zt@C`IGGrRU=Ii&pkXLi1s`=9-nthp5ql`2o+ zhC`Qh7D-ZwBc#&q>d!a4{Ytds${UPO6C1L18GYd)7qc|d@g@~_SB7wK^tN&sX48EH zwIF^!hZg*Z%C3uFO2U!$ z2nrs4iv|a25~7koh`gr4a7L}{71MM?9Xv+Zw9EFeoBO5R&9+eI!BuEwR4(-=)tB)G zTx_(cpx6PTOISLZgBmxmizM9C6bhD%@!?oKd`bdaBt^&pz}7@RlgBn|U0@fAbeoS* zM~S~P3yQN4_mj{NyG`ns7oJIMy6d`M9`S{oAv)X_O1jg0 z!+U2myVvv)DO0P=s%G|F5l~7ni zyu|e)@VvfSed^acbEBO3+~tASi^FFDu$?kZc$zHy<$X>=ryTBKmfH?{qUi@#HW{;K z#n#~MK}%cW2b4BBcC+6x_8*x zW_{F{d~;|AiEv{Cn;dI~ z{BQg5?35)7GZWRu`7~7fTWU!^vN;dAoVZf)5mftZ)d(@+#bZp1p5*fuTjV$5vdqgA zvtjaF2YQ-9OH1rcv+cpRK1XQWINP9d>Uk_X&359MB=svTA9!ha%_poA3CD3OA*m@%H7#B^Io&6` z7g_rJ_%`4o`p`CGvsrq8=J4YNY&-knPP5m*L0d=i>-1&6spEIS^v>@0P-w>rtIkEh z{!_O%XIu3-8n?SoNc{@PyazkN6xj_lQhNbThN2*Muglv9{KdwOe`+C1M0PfFg^clR z({e+IZ)oy#)qG4+QCiuFxe`p@HKlj$s3u86oR8RU#>*S}qOrhRYNqdo>(AGg2?S*U zEZ6ByzbVm`uh!j{FG7$1M#tQ-kThZS4j=pQGX+J?r?8#ZxI-yFJ!U~)3Nlv*!1$KB zYq5LspG<98DIGW9$tICs^eRI3{)Tn9?qDxLk`5z-spowVy3z6)J8-=QKWeUOazU(<0QrX@UKW z!qCg+s%glq5A&!VrL&Kp3I$)r5sEZwe?QPEv55*{v~!6gs4vn687A^Pr(-DB@YuRZ z_XhRg`VTTx;-O;*Sp&KKN6fc=_9(7Kj#yi*Z*h4NHH zmTWh>aHj3FQvE1$091E&s~TO#af#qqR9 zLP{hYGlN$)%LWO9oC}g^l4G=rMQ>P%Y&45dVm(nQl{G)*WIMUN?^_(=Rx^&BSo);{hPxDNshzWT0K;j{&ZjHDWH{LY~^1PRjz-DKD zc`SYY+DwDvEqtS^ySV?Em4RK#O-G@+pRUiQeST7VxLh9Uu|9CG<2mO{gWE%8y@`68 zS}oG7`JU?a%{#}WLCen%+_CxIM-uMlMxjY@r2JYrEeB*R8n(tqD+(z`o?BcGBklj= zNErttlg3x?_VW5~FiOeny#u0?w$tNa z24D9uhSKa)7jaPF$Zwc*c{LT~^C8V=C*C5W=dhUA)vJr1tV5%GSJ(xPy;=S>kj8F3 zl)*i9g-bE$W{YJ1=hR&$-fKTxGiL`Po|DIO+PM-wc9=EqI}OdO5~rLZ9$gB_6d}qV z5bM!=FVsumu;Ax+J&1dXXqc!SMRY-h)DV}_5NeW@dFDn`frp`so?ZO)9$i-7a1ucRh+;2!1BZ$pLBr$`B;{gm|y2LvcoJ;7+9IBL#Cvu zy$B=eD8@r5?0&;F&ZLm5CjsmbXl-E?R2wUv*IjZPBC>rh3odXS$5k;W3yJ%;AAWRP zd`OU2(?QcthfP12wc_-jAeKbTgCtNldqVKOceEfc87&W7YxDlXp`Ilr4 zRv5ET#>Q$uUHcB$1aK`(#*#LGVW@rLQz49h`g!W;EkbmoV|{34k;tt!Gjy>kz{`3# ztuJnq^XcVT&JG1TPIxZa#-Y<+e4oEj`5+UhGJEoGBJV7`_CxG63Ch^L zi$YeQn7=RXC+fZ|p_+w|!#Zg8;g}Pqb8hVyje4yzk zI?f^-Gy$GcaN85vPDiUggLDfDhg$|{Q6ih5YgSO|{Ovabgms^)#RBmH=kU$!fc1et z)dqW_7pZfSl>xU4x7AtQ4~O_Z0nd9JbjokxAA=ZNQEzsYzh(~y)D`7RdB=~Vu%)%6 zQgQRuGhb3csw(V%S@hAfrV$vWxm{Mq<&2@Ts_Q)I+$t!{hqa^g6D1a~*j6+gDsdYz zoH3Si6fAH)6&?~X^e1E5OGy6T<va2p>B9$nRSeUL5AmzI; zHpw}=PQK{J0rv>LhkGBZl@|DZHa3rP$-UI4Bfmf}Ayp2VJu(5xMSzWLupf>=VW-J1 z{-Vu)wdin~o3QyheSaA@(LNro92M_reOvYvPJRyI0^+`+y*4n(eoM`=*maQ!+O+rU zLXQ)tZ&dd!aczB@C7?S$$osTyx01jY$476yB_zD9&QD2|Uvwee5w|qZ){PY^QF|9` z7ic$^&n9okpR3LAm!bAHB|$7Tk?n%K2sk?RGb=4MzTy5YgiI(TnPAJCMTKE}UNA*N zZQlOL2?i?enA^W)_W+dR)OR9FUp*ka&P*prCDLr_8RajeE`j_ZVgY@?`^dvGsYR<` z{yqU?KsId&q`qyXZdzXb9t@o4IiFg-St#zP1xwkSnw~gtW$k%|C}%iC<~1;Fve_g; zhq?xks`@0uZ(QF`Sx9{cj%&r&53j_ME#@`DbU(y+Qn%6zNw^L3S z`u zgDG#`YQEt-e0AgKa+Ks=;Ad~?L#1;YZ8=+g31n9+Na$A|L*hxz9w|x^y@L1Z&wH>f z*fa*|DgR=mjJb5h%p_Etx}31wMNjbxdirzsPA9dr$e)|g5qztRAtFNGzs$CnIr=e9 zJNiC|6UJl(H9(I&UiPH;eZ@<>su32!hfZ$OeFl%t8Ef?+j&7}lgLMlsetfs2wlj%^ z?QZ7Lpp1Bz7AMzLcv%=5Y56|FTu?FS3Lug9ZXbK8^D;RF{{avYvjfzJEI6smt&5&` zepw%ArI-GqqsoM96EavlxqFS~$Yb&=zxH;d(-hDA>Zk8FPjrArsj#tV93_HW)l@gO zHs$~gwSkzWU@6pa1K}r0q>kCtKi!e1cB+%b*gu$wFm|htuQ<9d|EQb1qF5ZU($Sx> zf6eZmm#*Jzi_eq3gPRlw046HGjsL=Nri|$aSJa!%-@x*;8h-N0nauc}n|z&S9QFEd zE%ejyP$WK3rt;j={Xyd4fqRf3a8kXUd&S%+%K`sp3v z0_QKKCQS$)3UDVEwEY9h^!4*V!+ZvjC*fQGa2-Ge{);|c8mK=|?g6>sED=(b9`Jlv z!a$^<23bS+#M6C)0a9K?>dr&0G$N{kp0fjI7LNd9*{)FDf7;~X@^4aSE57LWvm0;V zJq}p`r0@taDey*0m1;q#8P4@QB$&FcRNrruAWaCDOZD z&+At6sEoWmAErL{tw^?W*a{ph4{C)sItB0HCD)apuT{V{LMw47h#(dXMg17N!LLfw z(r_EkM`Cr1dyU$>`P+b_!u-Rdm2p$gULd~o&mQwMR6iRH8_HTwB`nZRYAa~*qs8ZI^r zSYi~sQ7^P-DA!*AR7=frd?RI`m9Q>3MkZ26oHqK}Sm3N;Ebnr>;b@K3nhG3`#!~IQ z6m<#%L=3NNhGpjdHJ?OHO|lBPJC6?Ki2b(~hBD5Srq#rqt~<`R#Ant<>JRWx%Yz>N zVTP8@1`5g6o6Mu*fMkl(iIAJ^ zQ3pHOvZUmR;35Jz4c{nIBCr32@6{~Tz2ZpsE5M&^=@gp>c72I#BNt>z>2mS3%I1PH z6tRX1H0Gfb)6rmVd0Bo=;lf;1MaT`mNKcC}AZ)fiP+%ZXd`OyH&o9O{=tveThHQv634f`T9pv=@O z|G}8ppGmeZ)a3Ti2_N{a(8@Q9`tMiL=eK;d`U_f8Up4-NE&eLOkwW8cgiF{5J)}wr z`j@<=AZ!b7b-oQrikD~bUKzaVW>-AYY4uyEo1O-&T3Xxzhk0m1H}FO{1^raJ-&Sw^ zG+}Z}sp0Y2PgWWd{%Z#|#JgCtiZnCtPWc4W`^^A~D{f~~u4b_8=o&ha)*g0UrHk+7 zBqbFA$HkHP^vh}x`Pb*ok-i$jO+)FCkwNBOKeNtKC~ibi;=-_ze>8Um2#3d48h1Hb02e}sl>)QFGb{6jpKLnCu7A-!KZrr(yuBDm1*(_9&ku1D=TyP=6T3}FkGqwO#J3gbwaMEDFpckPFg}0$bv0G2~*}M<&l{=9Pb8O z;P&X3`%XEYD*%*f1#AXzbc6iIIS@q>z0{SD2pJ<^2${Z2HQT`+r+yxV)pSRVKck~8 zyRLRK?f*x#_*3Tm+lOaeZo6xkt@Jh5s>RwR-a~o_Ui?;+!qZT%e2-`x%6F%*U26+- z@~3#-?>*Mga`wI1PmnO4xW*y72D(_Btk4?C zf2LLW07Pp{9D_bEo5sBKDudA(QP;+hz8}azlj-8M^=Y5AXn)VPM7d+>5`=)1K~+uk zY+pHoWwxwWpUm!}Y6SW4-{Wicj7JMC`$)UXjV|{~RZ76y?9>GW=_KzZq!pktrzh zDQpRJd6zW`4tEx zxvd?vyGP@t5Bl>S*wwrbzsj%ff4`4eF+yQyyO~-p@BjN6%m2gok)lX(_tCyGs(%Yk z_#L4CG2Li1dFBtPFz~`%|E0oNBOFVv7lm2nnV%kB1)`76B5^o3#?A#Fc(YJAYNag4 z?luM*?Cvb@P^t*zFbN>4$}uryp(N;#NxvkGjAc11Xy$iQ#*m>tf!g#0ur@DgjZqQx zvkJHo7uH`GUL=0PDXaK$H#i=|(N9OVH>F!otlK~uw_z;6P0dB#F3dj}0tw53rrJB~ zLZn}Ja{>U0<}8#6~?J>3n)-5CvH$z(o}d>P~4f1jvK zCllqqS*Z>3owwA+qo0}K1AEv-TzCHS@UBca^=k;wUQQ1CFjsB#+W?+kLjzn2MT~f3 zH9j#pwE3~honI&;fz;zLfyI3}LAO7TQRAKFi)wvIs&o5%NKnrEDXZS`_sIhX3r(mI zA{3BS+0oX(z(iBDpNL-qEqRu6yXdm*p9klR0oC z7SQN?$Z4Yoni=@O#|nx6KJ&4{H4`aG3XxY6;B~%ooz&{rG-5QMBY$zht@|=A>(zaW zyfmtqAY{gA$6!oONUNDm3^7@E8XK@*V(4Ch{%brT zlCJZb5aL^^2JlSQfHY78!p%SGk`8V1P>`H^;u|n{gdOUB63=Ep(V zf#sKFH0PT=<62&vsJ#M;`Lt|YCx1mnxy~wIhVjbUqiSnOHM{Wl*Y!AY#}=I^>PJ!d ztu|7t9(Mh+3-CP52-T(p@zJGYu0x#Yb`*Pdm|u3Ns4#k%92cTX@6L2o&hJPRILqG$ z1lfHjtG7R`N0;5Ts(W;W+%RIs;v_qzl_uGl4xl@ZLw#(8b7`VwqyQr+wYGDUi5+1d zjv*U$@-4ER#o5Jg-a%F8szh;U?ia1aM#B_!E(ttYh)X-+91)5o0;Mwd0eZg9KoIYH z*X?26sEnV}=hyk7wyD`x-lkNR*H3s-)5TGUmkllcj;Q*NxJXA?yRcT`ZjZW;(ZSkqcVo>+p?p|_b8#rfPn;C zHi0Z!6r#oqjSZNsG(aWG?(6lO62A|fz|3Ft>}SzgabmIPfffG?N@6i@{XHZeq~WGg zBZ;1CG`M{xmGXz*fv05RzRltWv!!kSdc=I z&<$NNN>Vw(NZCRc0|c;)Pv(A=iQ)4kLHK+uP4J6}*aQuc;l{%in{|Zy=O$ewyTy}k zi#0!3r>(lcj8J;;oX=E+#at|9YFMISD6+NVz#G}9Yk)k9ib*8P`*D6g+Hpy?DdvZ? zb4h3v3lw>ozESNASrNCZrfSQwUU|&-RHi>dERwS$4tfbgEiDA4ikzmgH&x6UDq7Z` z1S2L@+z&#Xzz_{~JUFMr&-H2}ZHR0IPf^;5&y$(>F+PgE%QLXXF6hU~1iQ4Fo!`xb z0(iGDStqtWQJse2rQW*VieJ}}9nWzZ(ztr`HXsM0=%Qgo=MpH|ZWE`?^ruIUpM$+g zl#fujca|tcLGstdNN@rS{wNi{RzYf-Z!Hg4Df8LX3ymD4*hDpleQ?rxB9vp?+iBsX z$f#lEeDqFM+HK6r$dZNf=>0>0B{3o8`vxce7k^D9O{5{w?{0*pomU7{miSiqqvqc9xaT{`+;K-6-!4 zLne~5NNwcKIv`$|lmdSn-Euk5`Oj)b|3)}skQrrfIJ z#ZB1`R!%3F5ZpMQ*!me9l_l~0@N~fNPe(v{@&&$N21ZjG9-Uv!xwi2E-x$hemPK{Z zJ3_BaY6zinkAV4kBFAUxc+O_dHT8edFYY3Ln%zEMbVDIRIAXOt{HLvs9s@mG(-1B@ z_;3@HTFW}5#tZ_ws} z@i|UPUtm}F(b?T~`|vbc#^@}Q24BvM+a*&$sXW-s6u?f=npiBj#&sCLT!{#TO|4Y| zoJRc!MwG-$QeFOaud1>9Ure1d=Hu4kiUC#MILY=!iu@^8Xb~}x7(NQHni~2*=i<2; zN5e27`%IO;hw@Mi$O+e%DZ13jdaS1?vq1u?ciC7`u+eOV^k4n>?Fl^(07XSF$y}in z)HIk2(tB_q>lJts*cut?geCZR#YS^b3}y1#4_7aezi1CW-I}JWd>R3Y>h}5j-ywe8 z0X{nir@nF+=|^mOa^SE3ThIfzCzUomjEgO9H!goR>^aF28XHUILV(f!tME)CvB^BK z8f1(b$MwBm>z`9P*Q;?88u~WCYK#QO)3}PPv)M=B6Ae)ZvC;pYy?r_=`2U~==l zcuq2s4v!l#sDSeRD*&6QE+ZC7Sb}J#_IEX)Y;Zgz#u>ZVdMcLC1s2YqUX(R zC{RUlHqbZ?7+;btWm!w4=X|SU;vK+9Vh-3+)Iz;>P#vWOIZ!g27_05SLtyPlg%fgS zDpEMl^NU0o07YL*pq8|1c#71ZIxbJ^vpL=xm)YBsAezv1!10yGS zuOGRuMwc`R^uXMal%86|lX4Y%93TI*ol`p_BZjkSM`n8Z@8O@0pe4qN>)(X3_X3>i zNw9Zp;i4_>&LCsXUHl{frELP_uMSM#0EV-{9M3E0+~3SQztASrIb>Ym=kR?ZvJ8S z41Q!pXF0zvM7AFMw>N#m9#-w1?YW z7B`|tFbw)5b%#s=ldaWBVV;%Vihjb3Ub++52}e1f;J?@tLq2Zu!XPpFF469~kf1Zq z`BOuY0>-M40l$*V9;K*Rnc(u2++63ky1wHDF4Wl*%S?jB7wA*BV~2 zSZN;u7FRDQrI|!E2kN6G6K?L|JjZ+qenbL9clxvQQvOGlwh(fqRVCxc5)H5CJS zyU<4gKPZT9iv$Xp#Y9c=&ZUzvj*j{uwq$l@b{Ni53dE31e(qS5=;$aj;iysuxZWRoCe4BK={v+tqJ4Ex z%u?5yzW5Ec!DTi=JViZ>*DIj@7)kgJ<86At-r(;tq7x$RQ=DYX6udf*LcJq&D2j;6 z!!tyfS{9C;Oz{|>P9p*yih=J@!B>Q`uqtbsnLm526W0oLH& z=9?tVe(9FR4MLS6%O2QSpDu6~`~HIVHX8SErNG&YgLm5TwIm=q`~?s_R6&EMva4E?LqLyIX`NHy`Aeu^M6gJ;bj7UX@~iJ7V~OkfXV# z@$#inr)NKPOE=AEAbi4Ubm#OH&@71}4J7fU`_D9`r~eueyZd-Qu{uuz8z48(9$>25 zaDr#TX{aT{IPpryfUPk0CAsP@Y!(*r!H()7}+?f+WmY5{6WS@x=-+z@s0J2jl zt;8EUx@JgxU*Cn-^iUn9n(={)1v*C#DpA}i-vu?b2}qu`)LATFQ^p*V^Skp>7I=tv zQW}tqrj!>(B@n_5)n?+@Yt+fS%b`0CHetrTn9-`kGi%<(LraVt^>osmg0He|9nNQy zu~mii#nXx&o<0_`@d5+pqtiz(CS(A{ZJVUtgWyQpQrNxg?1(xqI056k#L!U*=W{zW zG+sFRXAdD0R`q8^n8Li87l%$tGU_?x2KgR!<~P)tN{dZ|c_6YiPXOZ(v>-`GAglAJG8rQ=Oxr{HY~B% z+UoQsYAtP$?R5Lu4WgZJQ-|)%7|8pcFnlm5y}|YlaK-m|Mb2xe0@8|g=e`1)4imK% z#|Hr%QE-IgwHF{Ej7xXIxA`x0DK$FWi>(804pfN`Bb{s;o#gaefi-~;Vq&Je#`v;5 znBNu5eMi)AfD`J?qy{N6^+cWmE+9lf+~vwPuP7s&yxw6r4I?Sa$Ir_mKRY^kU16o8n7MIUc- z^10{|W}BG*0A{}KeRFbXBpHn8ZZ8?pkG9B3vdbiE5B}8ntn|R)hJBYVOpUL94Sy)> zJ}e%ow)KdBERau z?fuE28YmP*ftZ%SYM?DCQwLzgDua`3U%()UPVE=EbopwVRe%tP#S z0L7K0$d7B0-7~&(sRM{{D7A5?Icr~jqWm(Q_#_<{e8K^&K!vRVKCroPECiEw)R}AH z>*l-|+pFjg^bSD)f0UotGzX}R&GbbI7lpw>%(X(m1I>tUg;@4@Dcby|DoNO#4ELYP z;|rvMPgMtJWn$s5-Mzm`41-Y)%H6af-m9uU-ioFjZW>MuqUPQgO)fqc#|1$ikq2B$ z+sm(-XmL+-(vg$ftmNL8;yddu7k=+J#K8>!rQTBh7*))_sQqu4*LQELjNk^EXQEA$ zC&vuTQx1!4AM+42%1fTQy-YD(93%GaA>Rh4UqW@Hh+a-DT*67}du#j+<(&L?@S8NfE;RVEB)eFad8qT?Dkkrc))aki!FuT4 z(L2q3Zc?AZw1#g`P#r9~lU)FvjQfkD-v+MdyaFzV^?0T3JD6a}5FS)0b$e3me*tEQ zW3uMD08$j)`0RM5oK@^?!|SRtGgUXyU{|)G9$syEJ%io@iAibzV%I`_C09no~ z&von=gE+_qIAdRlE}yT~Wg!2FUcy4q+@7}cJz@gSHR#UX^da?<CrhLQ4tRnb`Uk z#!&KthpVb1xihzbgJ1tyU6K73bo4b(sd%diVx#W^1)KyI>90ARm+wwa;Y08-H!r@U zzs>`VhW7bj)+#g4W(J1D| z>~=wwxeh)7M0+fmQPm%Wl!Wq4+_=EZ1xPZ?$5Qz0|)Z!(GBl=5nIdBs4~=6 zRvTOyCt(__DD((rJM?Dj;ss{L2XI`g-t!T0bN5WH1W!}bejwmlmG$2!&&ynU1|^dz zOqNu^7>y%G$ooU4?bK0~eKa%kB=Wr(PIA)D1z0ruS4_`PJ{xX4uGG@1C4-~77ngse z5n!_sZuUm-y}_kQzfp5=o8=pEESWu4?~Iyy%Vukk@R9U zAS$72Dsr^tC|&xngxAJ$`hm}*H$t`og|Ns^9Rv2flVfv%ErzfR@AitXiM9OICJo+mfeM(Pk(?yd#yj}6@b?#%D;#7|xPp;3^vsj7YbZ3<#cw*0fR$@J4 zSC;<0jP9e67;fO`2|Y)N?xT>JxQ)=YT2&j3WW}{!h5@}7zcuZGGu-ZZCGgF5tQ6c8 zbMNlrBOsr(3+0;P1BJ#-AARh}b-rFH_m+*5x4_ZiY%p2;56Q#PXFFEp7pSqV2RBYH zQfr;|ueBZxvj8*1Pb zXIGZk~k!PgF+0^ukMpMVx4K82_}y{dT1bjt&Okpz6)( z?9U!je)RM^{X77oGxqJ-Y|b; zRV|sn_TK2sWR0-A<#nx#BIhf*U?w_+(d5*yg-fVFtlEYJC*Ho5 zvJwX2N4n+MGtzk0mtC`$dVuNg(0QxQ$I=DkFe@*frb->~KXli<1Ew z_4Lbmo9o3MkGD6VI|b3C|52h$To5WNnb7iM3#U+KobKjVB@}jH#&T~a?RP4Z!-M65 zJFc-|k_(1ZMr=3xSPxFl=0B@>cKfoK;9U{Jp>7p6BTU+Np^<_oEArWkN4(hufPFm3 z(JoGM1iDw`oKdI-N+u0Cz*aT2!Pp34+cUHeNI#H!TcG`;AyKsK5eSYMiijPY@dZDT zTceUwkXy~D6(0{wARObXcQy__*JAT7ah^sCVhgfBGmnuHgF113L3PI>5M~YP*xK#~ zLDx?Ox)v)P!X$_m)?WtWyf-p2^6;qi0~5>__Gi4-hi-guyH_gup!N>_gHq_2K{3_XAs&05ph)j9#-2r$-(rG_g0U>(?UC=DP=Z($^ zeVL-y?TqOZC?HX^-2h%MVMAmtiH6)7>n)J7cNVXI;d}BZtpJRzB3leZs4zc%WIXzR zxO%IwsM>IU7zr5(>8_!XR9YG&Bn9a-Xn~QIkQ!RL8|jkn5|EPaW|T&{TafRWcklf_ z_)a|Wx^&i>SgX2tw;G% zuy2>&${{?Izk_WBV;6@ijE>~-2knk#WM`36!t777eW0F3ZNXuO2+J&=&%P-H3K5P= zBuFUQEFS<5M)o&L7mb38akF8hlq2%f>ML7(iC#q8fI zPIQL)Mlafzr&-aMfVW!Er+s(Z5Egq>74;xhHQIwV0NmF z@c(w$C+zH#s2O3`}xJ%=ATO>GZIplF|CJ) z{e{?>NP^;vqk{sC2f%TQ_ZQ*GtW6hf+%!dRoVt(EE(6EfM;`N>!oFC;U*kT8qaAj6U@KSt4wm4C*aPA7qZwdB zZl3F10n+&H#3Ts}KP!4)H1t+!5!XHPvQW9tf?#RX#ghvJh;6pHhaZwUYEjOn`Z8xq z@JFW}(RoMSOYcUzv*Od7{qrC^d>3>gYzj+8wA47xq(vKA3STpm=sTuC)C<80KFAei zSGJfR6~=?o=mL}}66K3XDJBdQ*x^I)P&stVwT@NoUvcY}8@UmWI;u*;G$tNZsHKu& zp1P$NWo)tXBH!sEsH|$~>P$yqk5c_klvFX*wg1*-x~z648+;}_!xZo;H0tAfpa%Ty zgU7A!n_S#AYVt(s8x~{?`-{9ye}*+}P?;pjB~_Akaq0W82NoIufhdo^nD&N)9}LMdZzL4|Ejs)qZ0(16 z630W!w{8ME>pDVP-8Bfu5R)Hpj8~y+Jy$BJ^M<@^%4~^?S)~ErY4sIhllp{hev{wF z_fStJ#|MXP@<+p z00f`a`Fzq63hV!2@@iZS|0SkhWz3@UCS{;ak0aHdHZl0){8$d$-nI^Ie=JmMMkE(z z@ZF=1U6tepqXkO%&VV_+ALE=9Qzahr4 zQhmOJw;wuuAEeN!$Ul`|2KKQ6fC=ZRz}h{wIRrN4KA8|xQ3ZUu{PIid^hc`mR@jTP zo|-r65z+!%HT_?~vSLEtB!z$U*iR+K@LQGas6?ynJa3794GRg63Bol;*r)tWduDn2 zKaWFSNdHSD2Bao-~SqM-_S!-XjMzn`(dZzqc2vxjtjSnmF2G6w5k3KM}D5JzyjWrv-SCDb1>N z%ztP6(5q?`LbCYYrT=w*atOWu2QLC3?cLp>53AlgaKR<(AkEbqi2c$%u<%d^ssVcv zaUOg*?`W7K8#?zFpuyQdhVy+ z>4TckxN2quD;l%HGU&6Uza`C=fk_+9;uG;6(PN+@e22N+$npZTZ1@O`bl*!O2$ zy>#)~LM9;D;`1+Hz3xLtO<`y_79(;{lfyL-M0A9}J!-7n@OqKQchOiPG6e?hG0{7t zu$reM)kW4wz;N+MqR;MEv-WwIliahH>(n9F6@)*jATSQjDfAt`ktJ_Z1-%RVLyN<)1F&nY7;Ajp ze^d_Dw-^s>4!~wSr3w#fL!w(C73H;%u#n!zW4Wx$?5MD`3bX!L^agx{+l;W$R!*GF zWuln-D;ZdvDH9=FN(ADj3JYsB4CplyXQ4^hFi0YJ8H_N{e)nmefh26s0QPRW@$Aj(eD`zrkN^tlKttHL?1@ejN+2TgrRlM_&TC~&3&ek@b7U0?ZBB$47v4e;4G1Wv5D*E>eG}5dPb!$Z? z-#z9ZxuIAheWE3Y-qA*^s-Ki4F7BTF-%P6=%N9Lb9WhV>BkVzVOKdnXV>Xz#07DQf z2R$!b$^u{b?rd`VzcjwM#J*(H{oqg<9A1zV6RB_Nf;yzA6Bn}xj2J&lvt2{=Q_%g@ zcMit)4shNN&FIY#^|D2q)I%2atzF>1E8a+^0_9CJwLG&Z8u9Nm5tk1tkE^%!02sY5 zG?2tYx1tUBd3?ujWnl^7J3^iya>Lbs%zT^~rryZBn`kJOkSgiZ=J*q(xJNZqCB+uQ zLVLAIRe8wn=|_ffCH$vy9X5`J2WyVCjep-Kypl3_iVwc&_YZ;j@)tuI2#+ZDY@hwB zW#h&b+JxIHh|h)YM~+%4vl@Pdto|0znxh<;(U$nrDV_|B0DG%FVbdx6nHz~^O=IH= zg*PVd_dtiW7Yk0cjTzX)O!B+FBTuBRZ>V!{4G07Zun$1uW68!ZQzamBm+lZpZ~kLtWK?LK6buy;2~)vDbl0tad}X-s&VWNRAWkIei6=1Sy~ePv+mY}J<*Wd^(@)&4A-CFWX%4S$58JbTXowd&}{Yg20th< zYI0;R#fOcKahMEo^}h2bX=PVk1iC^s&{G~!{ZRQoJ_UES2iGx-)v~epm${)n01hKA;^GC}N-sSJq#!k?cm-Wo0-+h-T8Bj-$je)|IDS#gNV9T08Y zX)>%c?jF~spC!b7Zx#^fW?{6LwT^IX=uPlWX9SGpa~*xgC+mVXi3Th?M5x};hGxpW zL~drT_n=S8E42w{5YUXBgnqiLs+Vy>1Py{4PL>`dVkQr&~V*0G$3w@%2f|<4Z za=5nFXr_3{hcC6uKn*C^XtS0Ef*ig4O&*RABC&~?6CM`z?R;Mmmz}%;hHwhT`8d_5 z&W5ClYRgTG>oay{`I6GmRtPmP-Y};C288BrgU9F5p)5nuHAsrF0wSw3z?-Q9l#x-m zYfDA3&(3*fqAmr}+nj==jQE}l5F8S9 z_Br<{m@zc;(#lSRU_O=Nq(Yl{9XgI}5dnmGf)+xU^D)K4w-^!WL>n z0g6G?bS@oAQ;6$O;av*`nQ#6aFU#XtVEzAdCq*C~0@t8aQOgVjC>RW^$)|pTK*P>J za2o`V3aCIObcwv&g~Ilz!KJ#Edv@w$HzlajL(E7YXgXrsAsm%AQ^4jNp&mk;!kasD zM59x1eUvx3P-8EgA?D6oY{Y`D(r?G&94&zXX!I*cD{$l1I;_dVXCr3qKx$sC{fZ1y zlj3mq%h^S))fULph^sDhf1Hz>sHw&MyUxx9m1KZMpQk7U1fGsyAT(MZ9mL6MFrdx@ zug8}`2{YH3-{+BsmH)^0j|e$>f`zR(StQyB8JG!$pp*CkR`@m03tF9Rjb|2z5K6P% z`(rT!z+4kujI{xzK4)QIlP2#xIiY2Uwu&Gmc!Q@MpN;OP&IqsxY?Nc!HUMD){sbt~ zvwO94K@yT#;%gQLpl^_1^wpaLNf(#s*p2|b5g&iJ;kv9HU{Loz|2A_2Qe{HX3nFsf zmlXDShDfb+25uj%bk#G0pfr1Wv!Xr*d}2pzUuoF89xuQ~F!+Dc>)Qvr0)+JcK0JjV z@O_VgLD35Q=qPlZfG_~5XPYS1ui=$Q!~y)67{G{+kfzw+qCxH!&eV{g&pJRkR|sr*tH`0jb|W%(|jwa)%MsLr2fM^Vvx>JfI;E zeA*&UOaLSzvXH5YH(#T_WdmT90__8Mu{o~DtSm+X^5(^VdzUP0FqM* zylL}A`U0~zM!$aLZr)-lt03KBjIL9vCtVcx5BR1hfYodT_UC?yDx4$a zP)=3f8ZK1=aZ`gKQ`^j$=UP7TWB_)kc!z7&9K{brB4~oP8*QC7K$ZbY1f_Sv{3-Cf zWz>(Pa9_@Rk!ZpotYiRT3c*s%a0FQ2N@?4MfqJFKbY}CF?j!e#?9%pQJ^VM=q zEWeS`XUyq7v;V-JYRm`GqaYl3EcuzK(bqF%&~XF85u5&nQ*wlCIT|AkCFAn*robR% zN*8GV@!U~(ZD5bzAx)5o-pJZ(18}qvk5HCkz~t8NsfOC5A9&x<50E-y^J_Rhc-ENW zEUN4^9RhS&ibyRkaT9U#A*z51Pl=11eBA7x7 z!BNJlCEr)pZo{10;tw8CPio|Qu^F(v37(F?@AvD$V!nx#Z50ft88w@z@~*>BPIwWZ z%WS~>4Wzu?hCUY?)r8?G!1)sNb|Qp++uO~7K|duE2&<3+92+ksh50bwkL#q-UGOls z!oP>@U%{tjRV)zu9sK{u!6>?l(z;O2`qI>B!zePvpg{-#kk-1s9Qpw%CmiTu;|TUN zw~5R9APIV4cf?=RLKzwpfHs0Ni#dQQG|VwbDiMa^hr%$HC5@Kgxd~1@S2@woK3SU{ zAim#d`;|yP4E-~Bz-T4Eu+4-AuzdCT*TilWwGcMJ8!QE`%o^wqwCX-+%k--0*{$`= z8Poy%ugN7Q^ZSdAkB+;)R5Ym`tvnL0?dW2Gl!5S-9Z{@&r#uX7Kcsi$F7h5M1$c?? zb0UzGGR}jk)}BjV+JJ(z)3@NL3Q(ev_Q*tt5lTnD_@{gRYB`38w(bqjYuihj?>nr`eJwV9E~ra_-u}8;+4Ul3IL?s;+k`P zj|4V93qh@k5Wa6b_v_X>DdLmCU^cM4CyN*V0US7VV2|M{7zdROC7h_>-Kjkz7yt3a zOBH}!Zvg>z__u5D9lp{vcG2E*S`YS3Qf_^C4di7dRZ@M|he`?T6xpXhDb6U`tN zoy^#pvzp*KQRM-|(nRlLY*AmL`ZJj!Av+d!u(89XWbv>5{XNunEJg~QcczNtCQ3cd zwxoeU__3Y)SUd|CnK?CB%_)L~4bVV@){KY6psn}qu>z{~)lbQP*`0LW;JmX<@p!ul zUY8~$SbhQ8&@49AmAoJ3xH^Wdy*Bm{FD}*rahhJBmXR9W$Xl!|3u}@z!+7|-=N>3e zTvNr|s|VOV&LW@<9&`nHdb9nHntFZ^o6(?4{!WKZe$(Yipkj=XU}MDC<@%o#^b6=y z^8w=Xytd%p!R)vx?JP1HOJN}nfO*{j{urjHL%%oAXb1}Qv=uCd9fQ{0XKi+Pb zI;2~>isYi{PbP?#NM)Kv*Ky@H+o%)NV~s( zEysBb#0I&euqmdzy8VfUhDVS~<|8=0S(4P%H)ziyD4D+(he;QGVTn)Dk++8y^_4c{^bd-iA^9vCU8G3F+Wf~J0q)1&AL;w|(BH|)3 z(q8?^kFCe@Hy{$ch&Jm#^9q=A(yT0`-+-L#w=sa?10E;mjutk7}oFqFX(32gU z@a#x+yTt9$3vELojWC7uJxEao2(Lga#29X&gz<3Js>n!N>6YpdBq4TmG*~A4sIgA0 z6kt`C$!srzL82D>M7T<>TN#*Yhu~PxTO^iqhRmi#SRhaOo?QAf@r=o_Hn{Q@J?T$0 zG{)3((0>(u0p3X6iI`rX-$FUY{mu(OIBj%tz_m|c3Ph|@);AY?XtD#ab$O$~*`r-9 z#CmeHflCdcB+LBPppptWaNIXo%6_7}(2bVXOP}9Q`5E$OA>8d<^m~?sn`D*ubMYc?5O)Ks*~#`r~^j!4%N*DLzgIn4n}#FtrXr%3EWE z)N&u!%l$k5;K|IxDD~gUixVjcl&}9yBy6AES~iPcmTnz3m`aXUeK*`HlYjgwWgXzUU+joN6+1t8{-pU-2DOm5xa38U<|tkeZyfZF zAdp!zstF@j<^5n1PX!_?!vN+^a^|Mn*w4EA;uW_!%_J_6mv~7>yI?y3v@xz1ATGIY z52V0)hY$AhWniLZZ<+#CCu+Jz4uLy_*`)U363$!E9>T$xsa9Y%$p^UHI7sM1kQ?0K zqpL|n)z?WSCrmR_Wta^jv){YiQ=u^r8>4&c-7(O(*`eTWEi&fsB?GF#7;l@)38Iq9 zugDxF(;241FG{8t9>@6&#^OBwcdBOJBjuS;g8RymKa*zY`*M{K%^WigwfC^+-lW_O zq1DX0#}&52;g&AbrZ7JOB8%@nw4Z>Ek==pBASc<-OGT)U3({I1MN743W~@5Xr|84L z2!_U$7gCqK5BfnpnjuNHTr%bfO;)WDa_De>{u$52v7dP*EzgP?3mx?Z3=@Z1mWS(~ z@5r`r+DWD{H}P_ycK;i^{hT|V+Fa4J>B47Gk{0~9|D&Xgf|%`eWb`R*F)7=z&{q+b z0zJ%dcvJDvU_a&a7)5j2re1M$vj{}jYY#&7&Ty2S+%8kBc5!5gqmDs*{EKj|lQ+Y_JQd71)jt|{)GM0$VNKP+IP$4t)86%XMG(o=TY%XVOs6J zGSed2F-vrWguQ70(BO=8YYy{ZEH(#?QN6Gy7s7C!i}+RZI9~NHM^Z!aO^wkL>_Ix# zCo;B7mWy&jtn2E*_^&3$fttiTVTJ5i4G6s>d;AO1z>VK%qlD^VzQBXU{6Z}%nzmm0NH0;c8(p7vWA z&{a;A@I;fvR+H2VEzGHSGz-}T0VVrAs;j4drpCW z+2!`L1_4JzQt-(qBuKBNmU$#1ocT5jsyEeU`a)IMO$P-I=%F_2w22dOP<%mjDT`yO z1khg~0#1ZS#|;QTjudtJM-z`OK*N^gANTZix2^@^Ik<(msJQWc2a&E*z|0mu?e~d{ zRw?)PZFSHwzQ}Zr9JaQaHF)<|DGPUp(j_utOBl7TB@`(O>Iixrh&Ba^o(xiXXTL7X zFxML_=sA90{&({-w>~LzWlmhbuZ-*)gK49m?;_@bY>In~Gp3^75xQYT!-wJ6Z*kg@ zPUt@(6kyLV_4Lo_#J<=7@QNYxrCwG2jQYEQh4A^cBaHajzAY2(}P z_x_I?KktbX5QFv}3e!pyF4#XrA?W$|x%wdkSPI^kXck6(QLazapR5qt7MgF=xJP_) zG(1b^)hfS~)Rs2`nRC7>11&6qdH#Pdg(*RxHDj_Y8w&EtpyAm}T+==F4`Kxw&foXE zZR-$;9J<`DT;677W^<2q#9^dg3qb+WaB|u}tf)RCD{pZ)*KA|3Z)uFDp3*D0lpcXE zD~q=eKIQ4;t8L%C$^X*=NETaXt&f7@r$q!ToZgCeNMFnVFg$(p zckZ-;wtAdKkIu)lD~yX`$=rg>ae!LxwO@nnlK%)*qr?-F;CAn?ZEm+EMvihNzof{V z-9t}^t9>Y)Y?sK+4pqa7r(Eu0FTTE9?v6RMzF*WV(u(`%Y<0(RHav9UJTiSb(Pa0} z#_peBA>F>uCrab7YdLluq4Pp<>nAC_J)(3KiWT35nG@T@HFUx9h+@EfPVA>b*SODn zdV)vey`Idu8cZu3L}ubFlu~PXtR1;=c#E$K`(>WYK9G zVJd;7$DKFKo~z|iB%`sum^4LT)?XNf*$Cq^ggN0Gj?J)zkcX*+Q*_PcjZ8{1~zoZwQ+2?v5Bp0775bds#B zKQC5T?t6g&PCfpL)cs`dSi^|I$q0#|P|hqJWt&GBje{rB>BJm<~#U*o2B^`}3+PBfgMJFF5& z3cZQ?dN=#UzYY71n>n}s%-UAuc~j{W=QqQO!FspkBOkr;6@wv>RWFuP&WbgwhRX|1 zH&7ip{Di#^V3eXO5sIlecM-tHGW%h=gpo{0+-+WhWfL`)<8`CAJh~ zrY~7vhjkB-W|xlmMBcp&2?x}cUN7O33KIr#GJp9R_yp2R>{$9ii$k@cU!?4ta#u2OcO9=qSvX|u>{{OV zgdCJ5T>19T0&;eRJAqzPM(Q!*$&bx#Ipm8lJW0tP!61+IYUO-F#qE6skxo6iZ*9)D zYy;SCpG|zbk3b{igs8YlXG&WD_v8UxFwNxp6W9uAt$?GA$wU;K7X96S_s>tx??5xn>ecL$z?G)ZP*%r0qtjp(y;x&YU*M(ubD~1>AlK<5&5Pif zQQbV=H=$UC-4DAw4A=2gNuHYn&AGFS_}nyw(}cP7TnYl-4bMWw+tRMTeo=eZtRCmV`>n4SNNp{SKLfS#USR_V~IOcw*et>F=zU0R^XfY+$*8FtTK(8Z=B-5awf$N z`3&B!U*>XP96mhul8|w!hwt$b*xZfKgo6!A=?(%^KbD3YoIOPBI|uRhJN||5_Q{U3 z=DVP9hBKX7au8fB?EGq$K|76Ym!oBG=UIc!F8fae?X`tKf}H@|wOvHExX;JwzF9y} zgPN;_z|hp^xi!fRx4~c60nH;@xw&&agxj-*e%_ah`&jf1kq0CMSQMzzlJjmq`OJ?dC7#=MHxqJLmpesJ!TxQg**s^akwacSSjvFJekv%eZ zW49`j4L9F%sXIGbQo$Am#0>pA1-<=h=~K?m*hyJsF@~qH-oWY9@f;PT&S`a%E*ban z+RiRlV~-MlPBI2-#fT`GqI3s}AJ9+wKbLzwm3l{kzChWPw$lyrw!fqsk(r^xy_aYD z?V77%VZ@o2rd%1NF+>!S&O#J?)1?C}rdLm>0tHTpBqA)=^Pk!V#tpQJ$;458W| zfJo~+$CR=!%OCBG^xvYx0s06)ic?#^t21#THCG^G#RvKfZn&ctJ^|#$FYVEj_q=8O zOlMcBYFgPz{FSlH92zsq`1V6=5dQY4!X5L0c#Y!>Z$Va@b=lt0=S);oI{Io)JL??c zjEU+;yV~U6bs3(Kpu+=DWVoZnur|aqf)tiL3G(65W?1ch0DTui2FkLe((eN~$4fG` zDYX!(F+2n`udlDstlwwQzsja$`paNGN_1n0@~EE8<@Wl;AWyp%cllMJ<nut_kg2R_g2 zUs47JNnCzEN6*myo^o>!#-&m1*~A)=LjI>;Y})_f|1u2I^7W~{Xj!_81SYZ+6%`3G z^v?>=Ele0klR3vnAQynF6Zfo_Nv3*On23989x6x82DZ$B_&EhUu_NCdrT4J4kAu#5 z90gR~0Ig=9tcok`bu0RT$LU7>o2Muv3}*nyNHpod*F5ZiNGl#WUibH|XcBm}tpGrY z5M&H8m>*vsR|ELT6@w68y?g(+!%rG^%s|B&Lb?@A7-7&>`ihH*TCz7@Wa^V6D_f_J2M%W3*nG{e3>2c(B1+&${#VO#88$dhsBzjT#lC7g+n*XFQ#V2*i$&25yfF}Y~nb+`&9X=|V< zXli?PEzL8KtC}vXk)XClO~y8<10X{gMFIYJR(P5~h$q-#5~>P;Ges$g4$CP;Ab?$z6wyCX|n9jgM}tcXro>dY{a^iy6dn z#9nQ*T(K9a_{CJmEoyn0Q^Rd1y)@M5uTFLW28`_%^R=y`8Evgv3mcJKS-Bev3s2^6 zvk}v~vz-QC9zNF#4x53!Xs2DsEiG-P-Wkss;r=aJ>p)LqB&uL+IqaMc?(UmPz5818 zKuXpGfW?tat*2%&;@6=G-K(GCr9JuQ!C669r@nvo-*Tc-RT&qh0CSit(yPi5d1}v4 z4w54XRx^PPMhO^1pe^B3BE2{C=+#!j@%Cb80Fs(sQTkN4cHZj=niDl(E=Gb~l!L|% zFym%qM_79$V%0aqZO6?^W@K1=d784RYGfk#zrjTcFx1*=5d1|z1?9~s#(qPh(&u*= zVOwC8risedMO$8adG}x$>?=;p2C|XZ@nFjeC$|D2lLG=bpue>W>tQ8ZjZk3nlWo_qT*kN31o zljF^7wFB~D$AJIy63Ctna&&Yo0`vrEvY>SpF{#DDl8;1qA79-YUHekC?7_uojnZDU z)BFupGMbiaVyM{=e6>PY`*)gNt9}|H!u3X4r2Z0{-OU6S<8P*1)L2Ra6@i&E9OrsM zNaE#EyQHQ*Y@Vy>ZQ#PQ=Xy@p9(wUU&nvYF)J1tyA2-#t%$SY--|rmEIsWah|C8vPA1845#qp)rye+cMWO-u!$G+mo@P#%*6 zz8PVKEC3@0RPukGT^5ML&He~&bn^m9_7Op=m_Ao+==$rj5i^IM#LNN`;vRv}NJG=} z_7xISu0M%=r~NzCwyfVg-@bEHsg%xhnZW4`U&7joJp8Elb{=i_ey)Y^s<`0dzP0w_ zf-dtHt-i&AJO)os`{QG*i?X~H( z>1w%|+4RZcAj5s+&*UGW;dIXtr?x)`r_oCZDqhy3nxN39Ch%3tR{M?4g*P1dr&*v= zlPhhd{KmF%Jo&(D^F{Z+T4+T7P;5RP$EI5BqIO)6M%~)O@x?O&pO92=x|FJ9Arx*fGQ<;%*a z>{?*y*hcWPPS{H9xSL4cZ~$XLqwgMo=e--mi5C6BMv8HK(I2AUQJdIv(UqrroA&o{ zO^7{T{Ja)ntn=b&SJ#SyK!b^H#Ed37v0r7ZVME`&3QD#^PurvRw=7l3ir)GmqooN6 z-Yt!^7(8^(F~9R`2f4=nVWQ{d0o~<=A$daj>(V z&573!r;sWHV_;M(ko1NTpOi#UH>nEhgU>)DWrU~7`Lckj>CSV}B34%S15DRP0{+3Z z?N>;p^O;>Qx_g&UuDA^h$ed`>X2_kkDKj2Yl<|h}-PnIf8E?2jke+aTGrjyG6`j{- z^VMMV@dF;G_54WVii0|^vbewf@c?qtamJajnds@zmj81Fol<+<&G`fDnvylebyGlI zydrf`b`E{}Q%2cytoNC9xN=eiFY){;Q<;?;@WR7?$>#NvnBnu>9-UMIrlcy#XDnXn z-mvw`zh1LBr}7O)+t18li84Yt{7JCSP!O5v^&Vy7z8^lHch-N{6q{$z_PIy74?F@? zgu}9W5@cgS4TCP(8wq+ZcXQ7y#F}+CXqsG7nY%3OKLoecI(FvP9`>~4;PHh9q?|PC zf1H%~gIgG}|;HIy+ZwEunwW?a*ER^>F=oF~x$?&po8 zyl0|Dbex>B>&@W;w}OCjy`0Lf!UI1Qn#N!72gimwVn(gR4g4r-GK`DIYod(9Rq&}9 zX^FIuc37iZ3L}q8>)XtdNW)sQp+7)YwsxCwB>#KciNmXP{f7dr#g%V>KiS`I85`e* zldUv?HZg?>S=?Z{HZkBbHK5Qk%$e$h#T3v=(Cz)Maz13x3Fi>Ti- zQm!ZP=~9E)9V9sa=_-g~n;xF}PRfja}lu0 zG&DwY50xr-jEJ4;e=Ak0s+=f@;xcFXFmzW#c$6lsr}~=MJC1;S-Y3K>AWxqiPWk+A z&-tF^y1ZHbqxc%S$Fc#t@4h^wQ%|;`7kO%P`_wh^3VVK%e$ZsIPdUSu=Pf8p2?=(H ztp3hYD=C*1jqe7kTvVY<$#VcvMa+&Ja|YBs%E7+eU!2u7^r&J$L61l5h^RY$vw`vu z^iBDt_1gj6)D{qnhF)Ky$Xe5OwB0Q3_R^QfXVH7{opBq{79?W zI%zOY1KanbX=!zA%LmWEqWx(|@@0XreXM1)VP46K)pxO8AZ9e!_~2q__TQ&$HR>Rs zj33e1>5t`!YZPqPioN9VR!+DGyG*<}-%VrteGBKTBmQS&Ur7^5IWlfMKRP~-wnOLG zM}G#K;JplOH%T&v5H!?2>@Z6xgxhhlKr@<;tos6h{dq`#dj`uHrQBN~}KlIXTz? zHnllb`^p9x>bTqS(x1O1_>FRQ!w>me@j$1=792kgh(MAcDWuUl=@8NM6XM7Keu zydZjm)1MKY2xt*)2v?>i&C$iJJWwymK&=^*eY_6oJl7YY3*2}hLjG+fm(j%a zyd&kaMla7oWm3l(kwv~r0m1bm#T<^EeV%S>c$fDB{q}!1Hl{s(ozfFmp3z2A8*Fu; z0R$B(TQ;b-RRn*_k>NP-gg*HZ3B48ia?S+U2|R#TQ_PUyZG@4>cK+GL;>WblH3r0f zPy#k9M-Hb=-lCB5@S*`Zr9+X=EF69B1_`1yr#F^jPApG9))X0anXYmBaoQBN$&wgm zd}I(xD)P=!d=wv%^B}+4ulgd!4qXZ`*k~&_)l66qd-4c0?0!a2SvH&1ys+ZPdrmj# zkH)Qn-ds760_W3n#BUto(DUfoK28Yb{3 zo$B+~4|@||7sCp;LY`MscuwsM{52RDK_}!!Dj@O!6SYT5B08;?|A;S6Bl?w|u6-&| zDM3mLktgIL4Jn@u;SMz`skI9vjclln?O!ftZPv1bNi+rBFeQs<~jy;;!6ZInpqc}|Ld(w!dv zkmSOV>pDM_zy?u6JP9B?8>ylpapV~8)+A_EDW%})o=|Srn^fV2vou&yVZO>^C^7lo zTu4Ty1dAI@9dZdz!Z_(c_cm2!@7n~)Y-OJ{$4Nuj{>g9*?xON!u)NB&efgx_2+f^F z%&qb`8w_mbUsxaSdJWy1!xqfx z5Q@qQOO!(G+6(77-$y@(T(BAN5}ne{7Q`-wzbK_yhxmcQ%9&*4e2=(v#p5|4T(3MT zZQ;0+jS{=o@4Z4M#9?uF{Wg=Q+Ji-OxwupgRtWrBmF2OW28Sa@|C5rrq zw~?X;A(I_z23N8&{>TH4TvL5!;J*~+!|#``!EXV=bPu$Bg|y-x$wgCa)LFryoNd9j zDr$2-DS3ayBZ@OHpZ7FA)tPc(ynt$suq`Iz84}WVQ*V5rvB-U6X=1ZI;OqWg7p3n# zRVbBBnj&W(dxg!_v8W;s1aotd;#C&Nu6Q>5ZK__R?*8PywQY{qcLN20Y<7Y=+QNVJ z8tt>`8;{yIy!@DZl=-ga;dAdtCvzZXSblKU1~3@pYJbIzS{ckEEa>vNeflGMj*G)P z&)oE(#(oVQ--1^3cr~W$(*QCiavzPXeT;15n**s5_nlw0A10o%o48A!irWMiSR|knlSYYYf3C%*JJQsnl58nT)xG*!-#-aB9p5X|{v?fw4 zGZi%pRI&jHGy!g$Z3qk$c7Ez&gRFKhaN&zxJ?Lzw*xVU=DZSXU*>Vy+x|v|h8!2L2 z-(S#d3)|rzMwgDxOA0MvirCUkjTrEAdZS{N?~401#-mHsi9?{-Dr(WPMg#r+!Mp`( zz+Y616Tas^EYI-ZCm??(1?>a-g<;#ago-MhUaf4)HO6( zRQQYbUXL`VFn`+=j8isRV=*75moTx9FG8Y@&Wcv{s=6bJgZyRjHkW%JHDRA(bo{ z1W7=usqB;q?B%kceRerb4fG8r0u9wXGe?QVvq4fHq~ZE&90CnZI4tIZ8J8amO@{lf z^z+&8Z)V%KfM7;ZTh&JwT_zA|+=@^<5}~| zLFy8j&>ldCeUUB`Xn1vpGuaQn{XfJSnOYj7FjXr&Y5H%dU*< ztD(^-LDI4Q4T1sdG;u?coa>~G>^n8KK~L%m`#z6yk>bcYT-`BE(YekJ><>rq4j|kP zcjtW_E9KvZ@$s+z;%+`ogGelbH%7LFnoZ#bA#!IC7h`yq!1n74)7L8;9|b{H3mM4% zlfRq;uIfI38FY|_XVF*;vzJ@GnFkg%sv6H3`8DVxY>cagg)2mks?gW>JjpfM===f( z#1G`Mj_`f4@rf2dUU5ABA?p2O(Y>c4+j()2rIo-OQaLCUgqRHBtJ+o^Pr$NCS!2faMM@AK+zOTfX#JohEn!%Ax&>c$ zBT%2S6-WoFgYEMECUx~~Kwqa3zEPpWT5mvli$P~#2e3rG0K(Ob(4lW~;A5STJwQi6 zPr)db`;4w=fhWA|n}-&xJ%ZFri(IeAosvOm^sT`ELzPhDZ^G_~U=hi%~;02WV>$LJc`8f#UXPfUpALELF{m=tU4T6C}${@&f#)AT&@q zOOG$C9rBp@QJE^8eMTclKi>w`7}ZTGyX>ge^jYhITyrP83}9^$_$opv2i<{yH1Z%& z8_9kHhAY#8J`l81s;GQAWBBpO58NN<{&Sfi3bQx-HYbc5%TJaa&X5JUl6wpAy=O9i zLCQ#4$c2j4<-8@`o8hl+tD&2L%nTzo8tA;CzOjocIaD$A2?%+!94!^OEMH`8Bhw6$ zj1>HUj{u5I!jpCoMW*Xz?^H>fr61gXg`_!OmgRK|V#Z||v!gzsuLLpJ!uz%Wgv9~3 z(hlgxrM~ZdcxW-)=MyF&z^tA%_QCWqI>QSXfri@;&0kv;%uC%t9U7rw@#VFP-7pie zvVutA_!6P#F5l@mYdiQg+Pj3fdcXYNNODldIdgAK`I-5r8S>T zD61|%-&+9#tqMK<6UUbSUR)}Yuk=c<*69~^mm{s?7R{d9i$B7Do9_%HHVi`y>`l1# z)wNH+QwqU(K;6mRdL%<6PC?*qE=_Enmjslw5s?A{}%)}A~t%3>&Q??hIpiuh&X>wyMznU zw?X*SGZ}+&v%n3Jl>nmfbfDRb%0F~8Vb!!=X>gu2$nWI)(vIfz7(K8(#IU2F7qtHj z>_p>hdx5GoKJOdkz0h&4hiGh`|nB=@}4h551EEh*Ru8580H)EI1OK%4~B0FYP2n$ zJN1q=Q+!O`ZB#XpZ+w{ld6K)ve*k)wifCZl!fUttXGdC={&cEV zzkL+5_Ifr&-YE?9xOj#}^CbyXLszXnO?M%tKt;IEqTAf4up!dAwz9?zgHNROF|>C4H{p|E}J_=__KPx!&h-uxx=hVm3n_^PEjuzc#BptZe2nKfRtu z6D5z^DEmf6uHUHoc@yKxD#P0jxpvvOIB%Av(lO@jWI7n$=Tjt7nI7Zu)#6euQ1KApF_6#D4AM z7BFV>wF9b#Lw-sM0nAxqr{c9SYzov40nhps(x6yf!FWQXC_tDTMDJAE)BOoZNHG$) zjYYC`5+AYK*h_2C^V~=^fpXl{5YE~jXg~CnZ43*K{HZUf5GH8}+W2`Gdn?1uGB;*& z+NkFrzC@r(=H?8vyt0b-m+kkeU}uWl-UTh)0t<$F7^TmG@yC5^{2egIoU;7fkGZ_(i(@m|*hAE$ z71eZi&+oBIJkGR(WNRp%#hJ1UL-U{w*b3>&RP9s>W9*`e>~Xedhnr5=j)NynTu|l) z=_Q?oOz>hr4@DV@A_zerkK@}XBF#heFT8e4rZDh4j8`FN0A# zAE7=%{RCX4sdc!xyV=3CA9kc#khFtla6j~OrZf{l2e#kz80`Y8A_|2mE?2o}%@UAr z^_XBu8xENfdOfA+wTw1Hw@1?mqUr$d1ZqAi3;ku*e##GhKk@)bdeyD0XLS3t?BI|t{>EGS3>CVuGt!Dyx&+ZJbW^4Mf~F0q&a+OB zb&}fKTu;*l>t1C-&ly5p$>Sj5;j-AptdBHy73%mMD|_IL96`lxLs=c_cQ);GY93de z1!JcZm`XDBXMqH_YkAwxQ`hWaJNY`i5KX|t08O8+GwT~&$yg3J{X@1H7A<@C1F33) zbv?}w$IX58+Q?F5l}=4rs>ZI?XoC~uu=xx6M2$cr@oCx~oxA+6&!+lmkKkCbVbFIY zkhw8(e?;{?XKgL?7qm21wyL;}|D)^aAD4q-=)#y;E_a%6jgcEcM;@gajVzg-I0$!@ zH^e3!T2HL+gf@hB*ZJRiy=KOav8(vv>YP z@e%LeUp<>KNN-rG0VLZ5lOU>WPA zjxfW&hUwP)%J+NRS$yB|82US)?B zX?}Ydzh8ox`tRV1G`+N!`&Uu7OZo=dY z_1LOf+c2@NN48J2i!Sky5)Hi3uG&-?R})tLNy$!8KvR)Efu=3U6P;^_Fbc zD%?z=4}V|{ozfXElJ*xNC-;jSpE*({yuMG6>h&s9{OSk&Y z7qa|R#JD24IA&R|H4Ta_&R7_bCp|`#M5}W3Cgd(m!P#mpQk*xLJb0nB?Mj^>eLMbN zTde{~`p>c^1_lmzf*cKc}W^iWsV(;O1z%)=dq);Xdg6wCFqo3rW5HO zeW-cNc=L40=AJ`_?RHKf@=_ZQP7lx$MAxKkRBeTZt}t4DSW2AL@RCS6Y{aitZOBM1 z4g4kVreA8Jx_{GVOyrvGw5!384YJWOo?A{1f>6k(jKO8*#zKW&Z{SvbGTXY3)6;^C`7|U6SlCmd!*zB}o-~ zdYPV&#{RjZ`12ddHWq!$jeZ$h*C#c=VMU(`Qx;9Q8`Fxtv98_DmcDnJ2ZU!c*|;N$VMu z+6izE(}c3$Tl}3pKqL1z1y7Ta?ayRmzMSaSvG*&u zse1>VaQv_Jfhd)B6g?{Xmt2&EcxBp@mqpo6`Fm^9A-m_QJJgr4?nfD$i58^0zfa_; zT|0Sk`3r#+=5*X6JpAg8L66)=6rq}1Er~D9TYVH=kFjnxDj_7-P$P(N@yniT*SToX zW{F0F%DW>a<6r8Sq9f1m>IOO77<9>(_SDeJBLDi3^fMmngW{FDZzAsys3$+xJT59~ zt$mQ4n}VTh?V7bl*jc#>Ip;`P#+lx$`?&DUlY7kKmFLEIhBc||%WPj^F@Yv6TltNj zspk&db?CDh@f8dH;P`2!2SsR(Pp6r!NWL#ZIO9o2V`rxSp8lbePh1q?&rEhXQi>T* z`gWx^=G-t8xHhE95d zbhp1)^bVA%Zh8S-9Z()flHPz?8O%?sv_O%cR zCbt@5@>|(|RHg;=nBE27+;?hNch7ryvTku!40#^c9nsT!&_h5JlLy|_qHGH#t>7=4 z?I$>-gV~craMHA|%b-UR+l)BMC*VkpW@ddz4971Pz$yyAaj32o#7cSk{TV8o5s2uK z&gMBik6zC>hI8sD&vZS`_SjiV1 z&aqKh>XzOl6A#s=^FLmm8Ff4e#wTotByW+G4%XETWd~5mF?RPY5&i-}aTo>HWcB%%zbhr$SLj@z}uRc)R|!yPS+KOU!r5y6U8Fplcmp z+H5Vkog1GvBsbU66S$2~V>Hx5IZKwWrI>CiudLUz%!@6F8EaXhzj8O#n;}f@QEZ2V zjO}5%I7~Q*@Z6Fsit4c$)7rScb}6W=V`t|rkuP(JM00$Obub|oLRGO+8eeX3KGD*5 zr=FzMy&~5wOJE#_Y4Z#upTydoQS+O-Xe1fC9W49w*JBKIgg>5Y3GFCq+w$JmDX#l|f9gOV7ZMQcEjEX9-rXmVO3 znTi$dUrKm(_h*)k1|m6sT)1s<61=t;z98X-_mt~p z{eFYcPU2jjR0AZya-v$z5wfh4AA>$R2%nkkBw(yff4%m!Mq;Gi98$JrLi!8H%UvFs z%~);5;&-H*D!G>j3!yw59vwmFtS6(XXQnPOEoje9o@rG`(`l`ueV0#>ex8yKHn8m{ zhodWxPu{+${n(^E+C05A+_Tm59<{(q#&^D-cR5AU?>BT>IN#DgXzKYeCOv<9c!z9L zBv9f*wu9=`rhT7U2VMYBdKvJB6o}OEE^IizdS|CGla>Aa`ZOcq9 zHulWvOEW7oMU@36mls&pkfz!=gwGH3m#)aB>?@As&IiPYCcEZPNo&0Mm^miGcou~@ zz4knhG7h;}S@GAR>8*C%_W>3G(6Xc{It5dQ#a^CGV&1lODor) zTS+W+`miTlLbo!`JgUb@ZfGpqt(*CweTjPMVgC+Oxxt2lrZ?lOpI!Zr^ik{h997Q% zz_e1r$uZ5UllyZeeYDnIJ8X@0puE<&IE|Adx6#2OHsm61d6S^sF(3vEWcZ+Q~8@dR~mu)cSAh(M{i{W3hkIL^LM{(i41f9|{rgR@8x2e1CC zLAOs?>@b<&p~%4*!<;_Kl^Np{<83Xe0^Q{Jir!R>udtfzzOOY{?!E5t z$J86@dU?@v@|*bD-ohrfe8ZoML-MQ#Vg^#=UN6_T_h!uT(FGFU^SLXi?N>AY z-4IQIMcwG|B#C7<}$db$h0B87+xXetD4;E1O83CFW`pKfG8=f{1ugXCm|j zXHEv#TfoMLgn|(B zDTNO4NcUBxk5pcyeZ9;Ek0>4%rW+eFEZ5^m91gQ_#PH#52S&sQh@|^E0>W&_i2tYU1}}+V}24VD6F;Q`4-iwVk12YRo7rFwpp7ytM3J{q*<0MN5C1Gp<}~%SxFwaQF{R=Q@Ji&lgP9 z#6u-J&C}*qf+DdWq$n2L6Iz+Sht>*C6w6ulwTJydnN3ORe@>Y2kQs!D+gPNVCpJVB zPm%+j)F7XJ#h~J&q5@;R4O1v@e{kVK6r<9*A@A;ZQ;Mm!fFlR?+`z>H-m>sWD%SeY z2`1!8*|$G)@YsIgB1x&>5Jz5Z8oaPMy?IGe(bnc0I8D7rZs^`-NO$$9lX-#^ahuD+ z>fvC!8&c*)pQu9bA(ub|-GBbV-m$o8xyfW_L)FJmVP?O4WNuWbdR##*+`4bUWx-hXDu%QJ);Yb*2eCRGics^ZN{sy-9R4)iJK)_S?ZbfQo$dXsykwIbYTT zVkEwQN$T2_QJq9wSyC&R_k4^k*&dd8Ds*8+f3`ql=CS4dwK9p?1f3_Vy;%8-k^+U7 zH~I2zUiNeUH4BQfRze77^i91+N#f@<&SNXVp}vw)H&Inw*M(l4+-3^2DHi2A@5FJ= zqO?=!wW`L5&JfrnC|H)8Zu+M<9h{~e@aT$}TJx_4`M4d0XGqUH-E7$us_06G;@({B zhOZJcC9?9~+X)1#AK12N{zw#{rZ0omuU-9FqzAojhAPI@%N{zMt{yf%FbeAtTA0LT zqP|CgLvia{ys(B7bIJYl#UWC~C9J+L&Mgw0S&*xW8BI>Vf_W8xn)1>sI3YH}C^gxOb8KDoUo3>C#?XduqrIn%n<=KsD6Jlg zye(0Nf_%l8R2pFCrm2RkE=m~JO?#-lHovzk{wG z7Njmz!--*bzGduxzKR3+9jvXCVAAi|nAXh%Fj!~UrH$|$R;${Ei3XQSPiP6+{b(O} z1XH-m#Ia!ua=xW2yFjBT1xv?KFdtL|y_x9iXvst58uj#tE7Xzm4{?sHCEsdOHJaU5iWPv5oY-7}HI*M9$xL=Kn^f!HxYNM<*SZ5Ylq7J)nslZ69oUTnP ze~D~X^vEPk$#SDE({bN_aPu)z5eLi#yrABV9sZ25RW@b1j+-rWp+5>tWtN#mF6pR6FLIwimo;(#W%CW2;SN1) z0I3L{3r-sgUm+0}& z2&d(5GCw~2aAh0>iAF)CY4yzqMqwLL*U*yAFl3HJ??*wFVSeYwa-4$F&shE+I6Gb@QJ^=}hWi*HLW4{1_GvzZH;bXgEB4T98)LRh9 zas)Am9nJSAH?dRx|O6-}r|{*-|5QXD;D;u<}8VE+6{y4u*e zaKe$1+f4a};V|_arpf>B)L;%F{UmIWC zq&6rqpvdu?)o^n6ILEzE7%(b#tx5CNcZtWsMKH&%{S8tjTi%EmoalLQ8(Ra=V-H`J zd(5nSL-F>{Gj1Y=X!_pFy{HK4j5rY)f{}CC=Q_@1r%#3$C|o}VPV@T{yQVXUdcw-w z>KQz=&biBNUnA*OVeF*{WW25vUsDzMLnd6*0zyNB#O6u1R1PU^@bA01?S=pDTG+w%A#gKlbXAu=37!6Q567s;F*u5+5b?BP3&V={ZOHD<$*~lO$@%E$fZl zjcIwTL7I?ba$BvGvCOz61 z*P?261~e-ua4A1wbp34bG5_XdizbshKez%@Q(OA1 zb{^afA7-xMygZ{vMf&;iM~%C`vPFwDPGk|&TU0rB4$$7j+wAAWp)PE&bo+=~TBzim znjFK{%R1vt%>ABos*&;ImEQE|Id!uVjY@_{<;+*}U~EHAYd^@1tQ6_6OX<(C>`lIk z%=mfnid#GpP2XSF3>ybb)gW`7+X1$xeyg&c3!6d_gaQqJ6iUEB)1B8;*p(4uu;lZ> z;--UI+cl8~hb>uazjCIXqyH+^d3BCTpyEbKgftBYH>xEIglyH~hllbc$AFa^)U|Mq zRD$_8u>uE@18>+}-t(r%Vt3uLr<2(Txk%JDgHqpBa=~(jcVK#_VKMcpC}HV~<{YUC z8|$tD+x0+WIQ7&Yvn5J&9xnAQjBws&%8)K~VdWJqL%pt4?Rof9dp-W|z~8nPY?|fM zY2v!U%;OsOt=#6``TuTo7;x)&HXP-mok%QElR;yhUN+hOn0P6|F7Grllws8Sjim4& z!u|$*UK~)7Cm&t-2De{9EDxzzaU@m@Msv^RP=RH~j^;e6dJ<7^{SrYaJb|$kP36~% z>9HS@QL2XQ|HB2C>l?gYu`+SLFN5#i6^QQON?~SW9amQ86v{RXJ~vpKB^Gm&nmE~) zs{=d<3*)MC@oE@X@~hN3bkKfusPTRgE6d0dQYZpGPDL?K&A{g&?+M9@@ivPAD)()< zijvz!)iP;x`FEFpZGGsMj*in@VFe}xsQNu-@aH^vP;WzqD}KTK;1POD6;pM~4nHsg z0_xKsh89+GM+A_3ZkW8ekB`+4mt=}cna$J2Eme18X;M~CERZ?8)YsdCZ0>s8fc2p- za7kx=Kpv9q1mel@m`cb?F5q&YoB1l?2|43u8X7{*k;Ky8dP^lDsUNkRZlnhhNi*aE zZd$IExX>HweYSBV_=gRII@X=mpzxHV=!jCCYwI znf<+X8m*o1_K<4~Bgv2n*Vg$e|2kf5h&Ce09IxyB%MF znDhi|@fPbRI#vd_6?UNRu+=f9%_%$77#Lu;izpssRTB5SMwB>r?D%3+X3Yv27oMlg z&VRRro#649T2g6xj)wCTXaujBw?XXkqTWQnb_byqh3Ybr2)<51h{C)ZQpo?7WwEkO z2ms%eA)Bg;d72luS_KFa3t}GkUpbIAN9j7(nEXY!f29YMzwGn2T!86~5|*5)VEuv{ z3L}dbHPV2m<3(0quU3S;FL~iA_(&}+=EjSaU8QIQ!l&c$-r63dxpwyqInc&Y0ccX< zbgXV18};;I9Di#xTP+d=1hwe%c)jK4#V_8achE)9z&fBXA%?pbG3I^=NsS^NZq_~4 zc7q1d7x15Nq;}+cA+vptA143Q9U<)>CtS#_|F47gH}1enHoD{O(u!_*!1gmt=64%PaS7CVp3*b z4@UQ4SIkjJI~eY1gXz$7GMN(r=Pq4TuILNm;%6M{8Qu6{F9F8psSyn5uZKU4$2l0o z3jc+ZNoRfR8h_)bSjddo$ounMXGBP;+~!k?!P(q#-AUYX-H(bpb*CRUym~_v9QG?e zi5y3d5Ol6Q6nS2T7Af7Aseb-zzGp1w!GumJXZq@@xX&$~yoK}^W?`IpH$yYxP45S7 zZKDXs2$3Se3rjPc1}~Mb-Dq6s5IIXf68YLf;xosVgX?t#Hko=<~T{*5(}^2w`s$%;5-zf~?^ zC9<&0OL~mpi{TCyo!3hZc2PNz!(o;~few6k9E~F~wC<{1;B&InNHd+A4^uPXw(8}| z6!PxZ*%$j{efHtO(3aRqU1+Qkw_7zn%U_S~C(3jr@(I!&x2k6wE!j36t^~bVDB@La zGyC8g3q)n~c0IO?(}ArfbFv*SNyRdy1q!%Aq={HsZr($N_MdvnV#?cGV&ZO)HGfVp zA}m2pF$>uAk{^ST$m*c@LLKm_R#@=x)!o*;D{gx-<|#y(c_MlLc+q~^(93LPI=Hnl z=EsyXZk;kRj0!zy7`c>iKgVcdPK}p0m{8b0X`B{4B#;i_AU zZ{jW#*53J?JMq7l(l1DifkbS3WRtnF_NybzT>itZ2Kc6g=H+h@V z30OyFTwdujJA8}!lOE^j=_-mYpD1#2(`PAZ!yb~XU&ulMsrZD*x`MTHj?^=Qt^RCV zx#5@J?#g?L8HnK@kgiY-v)$yoWJb|TfcNU%!4+(e0c&3A;`Kc6F;yrKyw`*8KB%i? zFkiDQBV3wT^Nian(p0iqxH~ag{qzk4m4?dClw&;;H!B#^%kM?zn6>qH^QM2~{o(bu z#WQcM)=V4@u}7E7?_$Ss#F6a}DONB@0q-fRC|egZW;C{pJo%*8h;#uPJ3jJp(EDDj ztG#ks?>-rsD0K^wDD&4B%&dvA?2dN?doO$0bljK3jD$H)9jBjM4lQ$uO);gbtE+}H z2UaV~HYA^8o*!@7ho#!QWajz?{jg_CV){5yjrU3l7Da9Wf=~-uWoCva?-WQ9V#|<+ z%3pvA_Z<3${Urq4hLu@c2X)}U4M(H8C+9QXt-zDd%BCb|rn$vS69l|f74R#uvd2hZ ze`i(YS7pl!w$HHBuf&Kupi>0gw{pEw`D^I&rAWm~p}?8bgL_zm zcpLkJ+`x?qpoV>1^9KGqs9e|XdKp;MR}Byuyry;oskalZC~eHt6VpdJS2sl_#Zj>v zssCjOc~M0_?F;rD9yZvy?;~+bhGbT9Rs?7m8l_%Yg?SR8Ym;?USo$cN$in||6f=fq znm>wRe6-2}#j@cSYdAe6N+ey~v5dCRLZG}--gW@w{$pO>bBL}Hk?o+-z>6kZ6_tFa}i#*a}m_ijLvL!>WT*%s>-E zI&nbWxdh9nL`AiOrKD>e8r(aj82px!n=UH)58Z(4%Ca%d4rM~wn_%sgu5}Am{>#%X@D&B~)CFtiOE~25j83mpj zMX#l-oAX4a>R|;(3lhc$xdi3k0Q!L|e4bT!YbRqXWOC*(4 zlM>Z8rrRefgKiI$mDH#O{T?{`yl})}4D5E;MO(d!9EZ?V_h`cDS=0QV$M38qJv;{I zqE_@a66=Zl9*FZzsR+B}%SA`e?PZkhy-&v)Is|yzw z7ezlU8jbz;R1mQ^q_-iul-y?YV^8wIN*w_u5xA(xoDNZ;;J+tu*)WS7Lop-fI)j)@ zZouMB#&S5J^Pm^*8m8`ejRbN2>A(0o2(eDIu`x8;gM2=``bKmaX_nd`fhb#&_K>=1 znH#M(V9nH%Jh?3cNsCI)sNYv{W{Ey9u8k-o%hTfBbo(mFk(4sq*?7%64X%t_8Ue#+ zE(57dL5EgYCC&%9>2t@C0!ai74;I`+YGLJKA(_@%BuOcDO%x}zOMcH&AY||&p9c-h z5yoh>Eh6kdfo#lR0Gyaz?gp=9)QaXYVE^(~T;?^PEkt z|BGKq6#r^3z25=gO{E;S{@e_-6gYJ*LFV?8bsJ`;-cy)l@z!j)fKqGi_4orUhCqAU z5;(b7e@ozBQVW~x&_Vv9s2q1Z_-P>{t=A7d|Duv~bc;O)x)1>w4 zw~3ZJ=K;_3g)sn{m7YU!P-eckmP5q5q8$gD0L}!t&W26kQRK z(V**NPIaX_uFvEGkR&%vFv3)cH--p4|8WGD z`>odn%z@vD7T5o~EOKNXOt^V1CEp%}pkCFu#PaXtE&tyqFBJk0EAAZX^$y;+;kS2y zVZ^ub!NNRac?}tG))r5|H%s$@nQ5R>UpIg&1b((MjvEP5C~905R&gda7J-l3+EEKr5VR1tPB|`u|->uMENj?dFQ7vHx>=Z1+nV2!5Xbcrno_zD?j( zMsmU~2Ny@Na~IX9y4<^2x3`1pcm6ea99f5-YcoN_qq()+e$KI>Ff-y(XbT703Sa66 z>RMC6G0DplxFm8BJfsU^gSD8;Uo`z47{Vv?;IN~I`#|fC=C%XHmx%gWU)`i9_)9*A z8pSzWTwGP^3vKz2MA?9*-iY+lD3Z5?#m^~9ey|lzdgH1=O}W5dC>Phs+QQzNU|c&i zn@I0s&u?b;$$JkeH0H6`C0TvU`ItGk`DRh-Pc$>r*h$o*Ayy}^*30mv(p@)Nud=$s zXcQ&tej8knE|rC?RtR>6@`@p>jruV4fBOuV#U0|q2&mF%fljz>pDx=aCE#6MzzlxC z;}s(e>$I@!nlV<)=Bq1SY8bXN`#^UI0g$E&ne-uZKG5NHDU6pjLa|@V`hS0I$T%g) z#S4t2>Mt7)ilg4b96_V3&cAZYr$HlI2)~Zfhw0^iO6-7W3SWpxYXA~H12VQyx86$! zG3-UksjsKNS0nztbav%0zz}$jbpwKGF^j4!fEf);#%PKeY{X0)5ObTmn;8QWZdH7Q zN?=XE9L-SahV2t=pD$1GQ-=Sye^JMZg(=ptsWDaP5;D)dlPGd^e``U;>OHtr>w&f? z5xY3IEl`0V=981FD~s{LG=f27ink;TWH6vMbSSpt+d+L|EzfgEusqa@X_v&*;W|*( zH^v|93vi&n=3P!#!yb3x*gpK~mvAyM+_D@I$3^bD2E7}=V}_y|oFg>ze0-phLq@jB zk_i}gFPF;u>u?MWRRNXO3pOaV6tylP%<7HX3-s??ZyFT(GLsMM{%2%6OuQ)p(_Q!) zJ^8ZUF}f9`F1=tii~xeI@%^d4+t4r_I7`RmNdb}Fh}Nz3$d22%hCOs?r5SYadn`rW z<|Nqql5aDrLZK;I10=jz@)@Yphp}$$&^p1I8Ikv1S(;qLf)=0MM@^1Gam-^GJh~~o zAR`>+MbA;vQEiDH;`w`Lv6k|i)b*ASU~NZ^=nfX=(Ej^Bl*n%F>CCl~6n`#KhzgrU z16?d|KT<=Q1I?04e(5&FRar#V9MjIm~~@wpn3an>#05}&QWrIT)!Ng zI;zV)Sh1ibm|4^u!1{X?m^IHquzD1ikX{zLsn21^qAx(1A!!?HLLPMZl~Cc)a%eqH zX5oGk_GBx&tVjcfA>^~Mm#*hnYHHlS7|4`Sw0QJu_CebSB)*ralg*Yb@B70E@!_eb zuQAT96K7*Y%Y*Ap7xs596u7Iob%}FvC$s7}AO)I~)M#65?RghrGdtA9UKJpW6dOJA z$rUAfanoU#Icx!|tT-r&jD0Ltbg8GP2sYx1f6$-7v){4e(X0chas+ZHTU<#*C;~I+ z6KX*_y8C4T+b{uO9qSRG2RgXrK)mcegaDPz7`Qi*rEJuf?L6q)AhH{G{VW%Sqc7P3!X;~uH_nalDwS=r^;Eia8#7^`u7hA*2FgWavz^!th=iXDsb2%?C0f532$et~qLkdUXuYr*XI z3!f^C==81WKTDtP>0hMc&L$EkeOXzhnYn6jV@cs;0Le8(-2DWcRMQro$Lis8>c1Ar zXUtB*IgkfbV!C$0M&lps>%tda+gr84q{oM~_H$$!9ZnttlBZDf83zh~ z6<_3Mqp%?>8~?v<4l4wJEi$9YlL#5IUa(A2h{RyI_kzLS6exBE<^AKfWI>I7R&j}1 zH>61JZWF3h(6$Gcmm$?gQQaKvmOrBSMffBf4Lgr6MS$$7h-1ppU$9kRq5q531f?02 ztHF~IQFV|@JX!$m{tw5Vxd5~~i?cF~#Rbk~qM^s>FgDet{VI_fZ^xEpPU>}L5QMPE z@#5{zf1p?N9$yaSke;Ut349ds30Q7ttx=G?teLi7hrc!ZXMa7-#NMcdYb8?e&lXJ4t zngfeXiTE!`v6S6%qlQB-CXG;Z`?Ll}uH;Ee)c(u|@}x+7*B5k6^DTb0 z)26SZ`ZLoGhX$!G|2csJ^quJq2|7vJdwUDeH$UF)A_KHn(__$*A3ToSnl%Z7@;6~a zowG2Am@7<~m46!e_RTke%#ZkGYA4h;3O>;Gypbfk!EB|`Q)rdG;vz)Jg%lt)+5V)| zI$@Jz`YK4XeA^Gs#rTiZr0PTGE>_!C@t(L*+@T`mIb7^Cqs5;g6%$qda!u6^#HO-d z`fIn{9DD*t^Woi)OnFkZMIxz8HDGvfEe*`o*sA|DstgyDp}6BT&*oU`T-G9+Ps;4s zJCSk|p_X=44q%m1$Y8%~MI(RnfVHxrmT6b=kFTC_ostc2w#+~cC(?}dpaam$f?*!o zNsM&ZK+`GEDQG3PQz$kWt`0MIz2wwT`n}OzTa+^^P&JP1g$f)ml?snS&c(`(NG3gi zC%Pn>&&`ky_1#Y?Jjh$ZPg^PYNb6+S}wErRI$z|X=C z{BnGv%*WJ|jzmD>AUObD6(wm;^v_si#ZLQ$;cxB9zrQ&_&s?I>enOvjG=kcL2c1VWtIRa=|Oq2!YWQ8KgTo zx0olXlg;pI#~i67Rq|6pgaddYa^jOaBZn`GpwXWnRg5^uS{HK6A-~{Hf@g^Xt49G? zq4wCDs0T6Q{Ki3Ohi6a(ZL2V3lmlc$SL1Rgd?s&wVmAGt-rRpf6bmB^{{2bK-_fe{ zj+G8BY>=SOZyXvgQaJ-8q#O&F4Jc)->)g2Vy(l}x@qE>;ogEi?@*#=S|;)_vq zM!c*(bMsJ`17?^W;plex=g$&1UJDC1O!Bwo325j$fJG~pZz(8sdW*&@zc0E0~Xzrx_e!zjouQxa)GsIK{lMok01aP7S&;;4ggnm#C zc<&1TfO4`9LuhCnA}VC?Ys|9?w=qhP}G(r*YoUc{LDrae9yzH=8*Z~d+6j)Q72;mW0#`3`%HRZWh zZvsW1ME62HCwBIF$dZ~i@_zU$&3S=%UItonxnmFrD7vO!R^r)Ye$fhBC28LBKM-j$ z4d>D2K%1-Tu>2&JtBxw1sd+!&5Tt`5mbgChUCfN{6fhKyR0l=t~Bnu@klh|#pyl6A5Z?AkGwc?1> zN|yEN>a!*Tjy;ZntBY9@n%e?tC{&V3FIbtGEX+Lm{jT-QSz@&#t!i z(c|Dn`%2pl&^&@^?sE>VMiQG8uzz$1w0S^rMV! z=)|bK)lMA{{Btuw~VSK_dfX$w4*SP{7ZZ5TEWUfS~(TkkD% zlMByT`Qo}=U!GegA^DEPjWF6Pq0Qv{#ZaK_>H|EC3=Zdphjq*}IdqYm&9He69?aH`18i~RDczb>i_qe zg1#!o7S?{0&RrSGUq_dRz59%Pg}irV14-tvuTWaCZ@ToDet>qb=!e9r_ZN2b=t*N& zKf{Z^P5X|6KSk?oH>gd1y+5yDSxWu;`{m#=Zr+OFQ6lxVB;2UsV%@kGik~R!P(QGT zsfpzDr5+!dhN^=35#M!9fJ^e~CW0xDkA#gZcQ1y*_BkDj$7cGJ)7T~Zm*aX1wF^8C zIwkgfNt*Y@#O5o7lcA||ZYDJtTXUs+FhQ_UDJ|y7?!r>roU)050;<$@{7H>=+k|UOGsV_eKqleApf= zRdYLKeX%ql6V_30>8U+CenG4AXMNgFN#sMsQfEe~(JS-4VFfZ>P#L%v(JhLNn}D1i zUO}Avf${}AdaXV^1f{9%50-Qay)2gc!A6g7_a|9?{24A%DAz8K_JT@_?x?zDNA1zK z1o0Pg;RJ8%&w%jq^WbLBzRK2y;s0;}M%DRgFDHP>5l6bJxdk{PWl4duJo~%dp_|GC zy$<>`lRzFHHp|tb!yFSIOD%okp_MDvDa1sq{d&R{RAPq~f99t>^lXN~dF%&mMFIv@ zFZRh{>)U}?Xc;y$U0yP}_n&1t@;`fA?!gKR-c(F?*Iifw#)FJd>Y9u|&e0*^- z-+RKiY!Ju;l4VWaDW)0I%<1>vlCB+vgLvDpp{-@tmHb7u%h2 z8_30WnXuiOKT_b`*No-a*c@E1ZtP65?41C)|Ewdy8EauHaxI5==nI+{)!OuaSn1P3 z5wi}SvR$}h)799d=}^R_k!86QK?3m4FSi=^St-ME+Q__0BKuK{kh=x*$CGT$ryLoL ziK2_6JlTID%vR}8);v61{cOMNJWy0SHyVgzW^0~7XYVkm5=Y16^ieY zlig9;4V8sMRyy+4(geRRq~7uZjdm`Zj=9Yps#)(c4kddPC*oG9M+GGT%+l2uTZfTe zUsaemwqKaXiV%q3=Z%?!B)~D6!8t&e7U2&Z%eZ8{fbV-v`!Aa!8nWwi9ir+BDo5Tw zq+y$J{@l`3!IKgauJecm?U1Qjzh{xbe?f=v$~@FsI|3e&4YZSCy*0st*8G?aY$J*; z+{x|iVZpz!Rr~Pv_lMdIjk6PEY>)o?u7Hsm&Bdiyyxg_2dRgnj)#b}_@WIsa#pHF< zbJrgPTvi7rYLnloZG1t@%Am?b)S8-@H9dYV%(AG^oOO`bS!?qvz4E_Y?( zdQ*CiseKX;*Vm|%N~MgN*X8`Fs{BjTa#bZ6H0+lU3^OiSp3h!4V43qDevHrEz-&N+ z-ff_(Ysr&tT+OIXlKq$K}{fef3?DzPZ zXMc{Z-#r~@R#-R*XIDPlm=U$B;T5gmiA&vs=gN#3ZKFQ*SsUdTI=jqI3KqAF_D+M1 z%|~;bDIW%Trkgs$q{+&SQ3sn~3yQ&p7A9pLfZ99^<$3gnbN!;DUKSt8GDFV9)Fqw? z&6c>r6w@;#!DQIrEmwVU38LD`6lucRDDN!)^N$#=HeJ|txosr&-^>ehzRCMo0VmqP zZ-M>pHiz7v6oNxVoTr8NFecz0<#M^F^7?Md$Xfd_ zGwr^e5~sVEu8PWy@q@#a=}vA`rRKe}luPEsVuv&139Ko{^zn<8`23zlug=Z>nQI8z z7Bac6pt|0y^d`hx(?{z>BZI+!kri|nasZrB?j00^_T)V2 zk6|3|pHm}Cue%y0mZVYxuHF3IPb0CeHYoliKL+9kqdMp|ik?1WWTm=ss&{y$?hLDY znCIJ09ZIQnYkz#6mEnfxBqqXiHP?^k!J-4cgL1sv#i_m3_fiK8WP>pgm*MS#aI4(p zxnBL%_4Wg4ZEwDi;>eE`P1(9{&Hea4+zF~|U7F4>xP}>`_R*VZ`d#Qc>sO!7MYk`k zI5%=PGUkx)BS95V?p!31lH@6oq<3}-}9sjjkIN55aX z6o?70Gi@`w9LnWhxNCP_J0D2&x(PH_$NK=}U(H0t?*6-4Qr5JUHiy zxk+gRnC*5h@X+{M5>I>dW|j_%@4C#T9Sz@p8Aa2RrQ+GopPBT8-1&+#RR&T^c=7tH zmeH0M$!qje46@3!>4uk6$O~PYCbw_J{E66AWu?6K`&_mpKYN?(;v=^VN;qF$+5C4| zZm>3>E;JkeYTO5+26E4HHd30Ju zrpaNUvzzv^K;ZPL2P(JEfSM4IBtq|XF`G#}f%Brd{iaAOOJR&s2kY0?U$lf~Vw1~a zmUcfgDl{qeo>nTls-pS}OE;y_`tu|bfr z&;#M$1uJ^Ju^)D*CpWZE{rqphHK&e_O2a`e{#RI!ruxc?C)O<&8*%UNb69!wKgk-< zhDPJrC8u4ThVpfz8b!s4a?SLdd!MyO#3$G*)3(F#sa4%Z?0&tIS}o#jFl<-9*K!w> zsI(V<95NWytp6)8RI`OY`Op-E2dMb0G}L>axp&1|q=-9D!~@Ug^3BJdPHaJ#tRF3z zPSSH}_~q4@FtsiBvZ*?4(D6#lh1}Gja0wyW528B);b*(@?>iq8 zq<`224UukpC*>zL-W5!FKv{nzE917*gh15p7nxOEHhgu$&pURJ~hl!xhF zI=0*V9eF)%T|e3MOVZEbZll?sE>TYJu+Y6@sua!u6JW9NAn3^SQV@R%CT`$rWRX?~H zw`RfA^Sy9vhPzibPe&^LifAm@*IvGP6XzEFfv)@E2kG3OEt%3YU*8GEuQTU5Om8;D z{17Hms=l9B&#{))otZ9jsAb~#>+FbRXj1sv(rzPr-Q068g>LT?dHT&DU&nB~xGF&T za;2x!DO`726K_oQcgWnO9zQ3jOsC_9$^5wroQQ=)kXb@&)h;Qbgs4x1a00qir~G&k z|NDh_$Hm8rlfg(F4KFm*GP?6H)M7dl20LLkV~U?OJPw!eqtI8ki{7triS{#pATvu79U{)Z)?kFyOUE>%3fBfP&=u{k+k$kL3{& z6R~6m4h4Y^Y~5P^DN=tdF_R9-D=6)m99(I`MlW%*BD1Zixf*=J&96{I_tYUUy}X<1 zKKDXm+{;AJFg^Y@%$2bZ!f9eB@f)w!llHSWk;cL}mnrC#Uw{6^dfAy+sao)R+#ScN zFG>V$-_j=kK_e=(wVw=OTVQ&tr1-zqN@aZ;2wfbY?Ozade1GPVWiQTs!en0yXvPTe z^9Cg&ESkSnejP9Q>LLz+MrT}5H@6?;^7o_sX}1t+#A(p^+dQ~gQ18y!)wGqSuk~vg zz4`FnV-JQyhSb6RhTl!5fnur|j}20co_aPZXNfdBT9%69RVkGeDi~m@{Tr-8?t8b~*dU6A)gUv{V`S{dX0TR8w zFOx1k{l2dcbx#n)zectBeo!6TJtZD^Huandbv?WvLuF}>soJ$-KYhSqoFu)Xwf;m= z8ZDPPi|s}MbW|+y-Z*)BrN~8p^3}Gv2zux;pN6s=>1)nu1M?J9T46kXzWg>-VP6h( zkiFpPdeJPg0OT+GEGGB$I8zu85i>W25>;3r%jL<7vM^ys6fu)I~w{?8if#o6pFm=!V)89UU>T`c|OM)Xd zo~|1%l$UGl2di%8b(5bxOp)R2Yq86k+mXcmBVb0Q|7lm=Eg#o4dugW`#+#e`tit_6 zi5>|7D#-UOU@Bf4!|PLoqfYa{2aV>K?#rZOt}Aljj^Cv_Z)p%Mfq;Yf`A1&9Nc?kn zQkXQ;9h{FVvh=*hjC=XG*T{)Z;K+!_kMoQR8knFRI$&0>alp{e4L&j6f!<9#0U}E7 zaQ;hDx$Q%Q3#m4&x%>IO-wk^bLL$|wb)T2+FrpY~f_6MiXk zH-fidUY=#lWt55FY|dc)3sl{Ofa4O>uF6lv!egr3TG8t6vLRL}w_4U<6@%7KNqV?J z5VYCL`a#8#w*P}-P@Opl+N(+OiObB`Anv93^MiEJH*22LA$??~xmxmhEzev^imD{4 zP=nm2&gV!R+}S}9CSA|nruVp8GG|9*?|?G>*xwNvVs&0?dKbyH)V+Fn+20p+@|p;} zjVjq`@@7Mq2e|Eq&S9I7<1i9@@yjwmyNOFRWZygmXq%RKi0O^Zdq8)v5r21S3oC9H z)KLA3B_ofpUiDITi* zAMXFjdpPm^Rt=~pdk9LDXmb;)*osTqlh+zZmt%Shysj-BfJi%iJmqfupM?5G0r3@_ z#=`-*lDlzn$9gxN*F7tiCQgTdq4ngZ7QqLqo~TCzQ5BlcV?s6*_y^tO1GCxD0HkKe zU-nH?phZA)_tq+SC!XQ zH(&~Zga`W0HAH+=pNcBC9Af zTc~WZ%Q_hmN!eRw$S8!!-iPd!5z;uwv5u9MR7zU^`=vhL@Adus{_pEt@9W+5mh&3V z=l#6z$K!E7ZuX!O){je;G&6qE4cAUfT^4LUs&uP<@KGGa&WTmEDah30kFmV+3)xf6 z7W|{o@!6Psg51}-wh`0?Z2W|^6bDJiJ1`&xQDpV=Q3_|@w)T0@Cgc85V+Q0O>>W3M zIvG<_*4WstvpZeX-_s!HtQ4qA%>cm?ZT)K0O#>1rJ0d_jMW28b8 zlaCYxFFmb!x5-=j?$eJzzB%8Ht;n5QmUuuNyWXF+E{D1itcg5tZe7)y3Mt>J?TlUY<^M6$8j4pg{^O}^P$w>ckI_vzbl3`X0$BH zQV;W=vYyPd)VXb=NHobe>?%M6w>(ub{MNo z&stj#VBZnlYAHn#j@xg=6nawgN1{lGWpvwTUZ^q2H07Hqn=^DfZ<}~$+Zw0+baZK0 zDdEVS<0R&;!Fb5MuBMT+TuOgF4ZSd#+LmS~C{j5OK<&)_Bh2Q=@-HJUDv)}8{D~l~=e4#s6q86b*?^j=0`_pzfoqKmyhLy9*9zT6XY*@{hx@%2ujL`@ zF=A2RmI%Gc(?_vtvUDwWumI=%BifNd{5aL>?JtMSq(&+p`BAS~Akh8YDyuFipWrm@&*aKlJrw`4WzLeY_;K zY&8!*;dq>J^NLm~!9>dIrC*9=q1(bK@t+D4QFk9q%Uvux{`NG{Ah=Ju7q^TBJZlWB z^a!dtbYnPUI60|?tA@q1Lq0AQ-|}|B4N%&2QU#ANc}@R59&;sRag*~pyz~I{gneA` zy5w8uhpRmk3jyjp(Id&+iUZ8iQK)m}dpyE}B;KNYS2rIgo^{oV65!P$k^CG+s3uQQ z1TpsKjA>Kv4KMFHvk`=T*p))bd2a}m7{{&Dvm`ZBI-OO+;fFG$rQ}2$Q?n#pe1o^| z!ihF%Hxj22DjijrQSe^ZB@gM|bRt z2Z=co4h45|Oq5%Neg5RA5O)bS2UR(IGah5PIlXV=IN*0wuxaYNBH`CEA)m`@fXetM z4^V>1w%&0)_r&CvPQh4@Yp)ScRtIn4!7Xt8wYRWpec;6CoXNAWAUL94x8-A*6xU<_ z64{@Xtn~PSil2ui^{Z|6P6$>|eVhp;rKV~2w7mZQ(G~2xv>3&nVBBVyw&XjzvO{mK z#-Z-yht=~BiZQgt%UwQ-uxG{Va`7Fj1chiN4 z!@p9J3XV1!CNE`HJ-UO2pEx?M(-=wzM(D&wA`<{L0N&t<1Q}<&r#IF4N0+A577M4w*H?^8WF?(?d%y zX#;E8#ymEr(_tItI|9B z<}(r3(Qe@J?0s&0HcESU@=o=95-DxQZA;fhNUvmDhjD-Rw7uL&MeDtZRS*g~Ti&l|p2`Km%*S$lw_HB(sT zS}BpIe#fg?9>4Ot=rLdfnS;EH#+?0au9zBbY2Jo$d@)~B>tSWIG79F7sNRA=sc$n{ z;(|#Q{kc)qAdAvg!)JaR z!W}z!)|*|I=d58AWYGebgE_MT%F{W`PEzvP(;c8Zo4M=z`XX$n$idC;)`LjrfO9 z-nC5=`GF7ZJPQ_nUjQ*>OWgvF`8AM~Yx^5ja$SdaegGkU^($$nUsS{7hh|tQ&qpkb z%jLc%`ljf&bGIg0w#wUao0Wzv2ZoA^E?pn}eaXk|Ok5Yjfz#Y0eXS$Gy-s?_n~Ubu zYr2!d>Ux-|RaqJ|;P2)x)&2qSsdTF)q0*x zKCFm{r0X_Vg{~xpb{Bi@ZL)|@v3-fxkN5hhs(h5~GFz5(=5e))k#l=I0^$sl7Wptn zqj>waU=wHui@=IK{xHaepBfj%W|Y#j8UO^*EYM{3Ohse4F<0yfw?utz>8I>>Uz1DP zANkIo_q2aQ7(IC-xn*N>Lipg zs@QZp7mAYR54J``v{lki@NQ&PTQN2|x1{bXx@ z>cwX!hNJ1!B;TtQ_H)hN;`cR@;w6t#>3uV1&Tc7vN42qS%%9`IhG2?r(uXrE_}XYr zy^}xEUH=Sbkjd(!j7=Yfg>)YIe*S0WA-Pa$Lqz|23pj8}qtZB5NiK_=170KlsDCf2 z4pG${S%B`OgS_<)Lh0#iP@@lI0p0n3nWkL&Y;#7Qi(Pw~QxLP2 z)>Hg|hrg&Te|=W*7BWMG$!c#xs;jS#qSVFPsNf6d4eI#xp_?2No)0ZJZZG@tH08V_ z_wdA6+;iS^H?^@GSo#XR&^(z7{O6E{Ae#IckgrqHk3;{y-S5bvN#r!^ks2)-wmYE5 z2tW@U2we)k7}A}L50FIEH+O`=(%=dTl5fouSe64~*hb(6gH`(qU?@CG&KgW>0HoY< zIi7Ud=+@yDdW~z^^o6v^5esvaKaW^GQp{*Fr{@M6ycR1yWocd2%wS(*>$}avpPuRg*kN(eFAL<3Q%EHrYE_}e{J7lRP zS{MHp{c2ss5ITSM=f@tQM9JtQLpcgQG;8Z~;I6MP^Y{y=UfMSE+|Bnd?c}_vrQ)u= zDQ^a&v5;0n?(aFtx-1G00OpvI_ulY3YHT*xU22?H6!AcTN=tkxDJax23}3+gh0Zu? z%ue)y1v&6O(#3lxs5L*Hy5{M1p~9*;#T9xY6FB6Gk#TFQD#Zue+Ae#$WXR#~`v5)Y z!TXSEco5}3aNL5O0(MkpfAyjSu#T)Mf=$2aX(~w(NkU~p_)Aw6%QJU>-&ReVf?0** zpO%O9^GZQ%!=`Eord6tK1sZYhPI+9Gb01TAF!!3xXxhYQpNX-wxZ>~*SCitJo92+Mfu4N zNJ@oRMJU8Wb=qKnj*tfWHj~}xgVK^(SXx4#n~$Eb4L*4RnXQu``6Qb~CVAy6(v1?3 z%h4RMEv`2>F1I3dc4iWzTP2pXdZOg9$ez4IeDJWJMU_`7{P7od>q7Kh zYdYfkDdS{Xolwf?!XTX!LghfyUuarOo6y)&$P}}XIt7m6?G8 zAAhVlB$FLO1=zR8FTDY)Xpg^4GE6r9`;1GJ-l)+y*~F!TTxE_DO56XlHZUy=1`nkA zGP%?i{lQF{&1dA!uYIEkD&}|v;Zs(d+1VKGm`c&Y)ja8Sua{=$o7>zg!%8ncVemeC z#j}eP8Uw`~o+hv=yu5QIVEMj6$T;%Q9;K=*+dcUk${ha_%CaGp*%^)-AIN6rv`zl& zIx4v-o7|`D>&>5Y1&SdA^(h%>np_MF9939-86*5q3+!<}y^h#u;CV|?nWo(_+Zka{ z2aEheCkz$J1yz^EUUmWrqtnte59nKi))5<_6sb^sq$Mv?ba5H((W#bKr=0G9a5o`O zeV#jYK)AmI^#!sNg`kV*VGcS5c}iF0A)x)AQ)$Dr8-V<5BI6Tk=TAB9P4V^7wufDi zC>SE!O2vYC?<)Y}RH426vuL{ww-E~D-K?iUM0#zZJHniU_!Y5=TihwV0amLb9cDHE zopOcw;6t?^!rTHMP2|}|DHyV{+&i3AR`P?CSOBxgoum(9br|pqM44p#C1C| zDyZz_gvwI8geDoU03BHjjZdiY`!3+N_CEsALv9y2Vp&KiY-IllMC*>r+q~-7YJ*DuaYGPp&RdOv;huHJ<60NTWg1S=|jS4uY*HmKCQ(y5} zrmH5G^VUA8UHY?s`_by@nX4~y9w_*Xy$G87+-y_z`MA%A=JapABRM~O?l%4TirQeY zXS@poS4z$6iu>na?6`Tvb!epbV8g(npOTgJG$wtLA(hDWC8!8bVZixX3*x9b`_+4! z^Vg0YI|ly|ZWZ|L1GB$O{q;GUnas!I-M@080)tMi#L$X&%R!;a+{<1GtH<&j5L?NY7OuS0Q7{^_sWGF zd0y_IFUg4SL?bP^?g=R}ejjm5dwOLyE^Iq64bm8DO`PK!PN>jo``>|>{<8Ha=;wbL z%8Hw*ad_^!vT?1!^W5r!4R=L=utmp$H{G{e!k-GUzJyA7e*>nb!?T>^`Xkg=?j;in zu?cBBeI84wtqtCsu^aLe@8o*!_HLyJ>K1Aya;+D{+$x8h(1dl=64Tt*xKOP3pn#9v zJbQ{Ejwt0cZ!+W1PH&ycJZ7?5rH%3Ur4k}OUX(n#bwJ?Wt(W7j3wK6)+Rj>h zHmJYTv&{8j*I-H8YopiN>tl`x=4h+J;VkxcPMFSp9P#7}b7r0{RwMfO!JNI5v|(k$~Yj~&(H8r)t>JhzDhxN?qg7F{1Y3GomK@{Ua5se zMWvQf_H=g5Teyns{uw?p*7PRt2%Dyu9 z4G$*Skd|^ZmQ_;X*N<-*;8;U__AZAU=1rmN9%tsH)w30@gzc-*mX1ny!2w|_tN6(e zq*bNx=Ju@7FiAuQg+tjujjVf~vsvtp;8|tO`rk5w$ za|GF`6zJar;O{I$xb>VD)2Ituv}2Q!2ZJ{CnG{~89ZzN%F90)32a9uNd4_goDCnpg z=TY+-fbkJmG{5*tM?BppzK}vhnRM_$37vp>;L!LQ*B`EW4RI>$Wfmj>=eIY#7vO2> zvsP6*JB>ZG7ROe9w3pOVxLCTLd+4;x=Y;QGL`(hB|6>k0$`QHef}k4g;ZDn&Q+2M#PF0pw7iWfR${rr9yiLKgA?|t?zJ-oy zzI_V3*?m9uG};|| z{p3OhSgvRTF;zO)l61rqyxD5~wk(2_3|reC=w{1$pagH1kG#8Hva`?B#MG4k(9T8^ z*-h|iWKcGl1<%B?>Ihvi%%tr}t7_Yj{yDyUBUjg_6%Ms^ohZ*bDiv@Cm>2>#E)j4v zXzE`nz*F~$U$|#V!Fl?ImM*Kx%WfayIVI@Vy=)X|d=?!BjmEEz+`vjcx14WL!S7QU zr*Ve8r!6wVN2(tCuH2c@wxSUBp=3Zp?8&u)~z$_xY>;$@uodF)DMup=S}?nG55!0xPm@YhzZrr zZF1=g^~MQye`u)CPYPnU6L3~EYo#xv8}Q2r6Crc-^QQ)U*7wJfJ_+0Ah0C}$6GvS zGlrNe^Mn*TlH;!*zBj3cU)qoBis+-3SOS}vuboeu!_@_YKOgoofiNEcGWoY|j?n$` z2@KmdzDG`ruMc-;^V*1KV*6SRFR){V{COzs;>f(NF*o@iC1Ls#As?}E!j&m|<0IFi zeY}*N34Po)arA}gi)OW6);vv)0`H|HhH!tLAJUa3lQMI#`r@$PXOr2ykT1+7Yx}TR z(tk(HvQaE-<8wGA3&bWEuC&#mPc!fw{UQ-a5&=SmheZ$MCRqEPN-`XQ7g)getl21c@_qnfVpOX#bXX!TaejHj^;xY9QY zrBMlBf~4Os5@%m`mqg-K`PppFsDH~|N-$V>Pvo^1v>Yd=9ir&;j_W;9T{6EC05Vbw zDGIq?T8*DUO!agxVuKfR{+Q>{YL}c)aa8;>DF%E4q;hq_{1U8~*1bF4S=a`NcgL00 z8t@h~SrtLw`&*qa;y246AefMdTp&wCnO#k`qB?g$ zX*aDueX(m4B8wXdw7#gj?tqaQ{2e;*(}IGe65z2|FP7aYOcFeDTkay zzb4da^gFsWI!3=%rY!o{BGV(a5}SEtQOaW^-F*b-&OCv0AN*~VGA@j~(idlQIHUjb zO8E_I9krRY(faPAd_=m`1oSqzcaQKv25~NWlU*xR-lRva-YWB zKYinU9vvQ) zPA!sVeCZ;bj~znU_WZp>G`Qb&9%cJ57(J{}ns+UoOp4G5Nve$s`&-|X zU2*B~XTF+@4zo;6F*RdWf`1P~H408ee?#dlK9ksyAQVE-n%7@T77!F11hnPU_$au$ z^S0gEO#!riM`Jk({6Jda zd@j@sJ1h#2Ff|#xqaBU-dmwy1Kq#8Q4+#|$JBimMD%CN3e8j{OdyI}#-caJ%rc~Q^ zaO-UAd>7$(v323*+^l#114b=vZTg>`p+D~a`F8i#tlah4a`QCUxW`Z;nA}>%9P)!3 zQC3AO$IzxFsf6;$Zli-EHbCQ*4wKVMoMum%VR@n-uuY!{%#9WeVP>5QF#6^PK3pVp zWLN^!^v~D(z;NAwSJ`VLEq7k=@9D)5N`#@>ZeHbT@UIRCBkJAQ+gU8K?FxMbiLgSz zr$XmceSQ76ua4)Q_I-5=>uKo>TY^D8@N2n{fm-p$q5Gn4ISK?X&G;j{KvleKnTa0J z5b~F0h!!S}o%VmVI{7#2qtqm_C*R5RTyywLMlA3^m6Fjh(Yn>YCVi#z5RG%{ZvSs@ z=0AeLn}gBEG+zV&TDJ^D07Ke#hz4sjiYs7=zSQV(B<{IV(0)DD%r#u!a(hznIe6xZ ztpBoPv>FM-IuqSlsI~0nk4MD{g&=X8Ma(Sk!^(hAHf%)w_Bx0Y-*+dj6hWa8-<7aJ zR=u6ccw(>!?J(OO6v*6wM4hFPY*YB|(*bKm8HG?SXT`@RpzO7S39{++3M_wLK{q4P z^j!$ek)eNT15$~k?7qRQ;Om=E-iv~t_c%w8n^>is>F8xas*nfLSBJn+`ioWelFQyV z;LwYZ+7QypplSz5{}MZo6Sea=E-g?@oa`|W^VDoWSjJ&?kC zFf&{^^6xo)JVyc!cKIOH)~N_^pXCoOuhps#qp8KNY3dtd1=Z#o-6lo@saIkdvWRrD zl|Xgn4H^`;fIJXM>3DnB3Ym(+F?jZ+iU=8lfsD&5fqZkXof=qD92}#Ze^-AnDve0Y zA3E>S!S*L5T9eW?kv=LAdPjBIZ+~CPzTR$Rs*e?2%?z}=tj*Zy1lVeb<$)O5O1 z*YvCek${TxKAaC_%Pd6~Jr_FyzhNqTtl%^`W|$XjaWmla_OH4m3eoTe7R+~lv)4ts ztO~yCX?cxuV$JY)RunFTtD#qAAfXI6B(L~)dD+v$sFp0DRHWUU(QSbe`wmPJ20_GG z2<}^J>E`&KkM*cr7aCn&p9k`%xN;r+434uWq;kY#F{bJ9Sa1KLOO;mUpb0C$;ejVI z(WmHe8pZRrKpEA#ycS{u8Pju<{$|~k zZIT{qCJwMZ@PvJvYy=!!0~lp0GEruYdSXrbY* z0|oI*E@9+!>A--OTqWt1a3vIo;h-X3ep*&(JaHTLY#PLaLe%#=|Kb5fSSQFFNMlTG ziyUu z=j^;nPC%XW$hE%RA?N1-N12+uW{D18I7oey%uo*lR@erU`JhLUt~de-VMs}XNP!G+-_mm>pX2vOp*c2U-fj+Fe<{oC(vqV% z_YHX7(vNGOKzn0cJ~l!8`n~giziS?qWJo%A`QsH?h7&P2E1~f=0APt_Fe*t0NY4z%TAgO)s)iHi8XE{6u^erJ9%IhcKg zI!{t5OvIa=lEQa$#y5jW!pbbQ494r~846JkKG8`RI~M&qLMDS`Vt4vz6DedXv`XYc z2ic02I*I4AGMGMtlXK!HMo+c}J!^&fI?J8BCaN0q{h+><@l4K(N6h>K4u4RKcrUx-aWZ^%s3n( zy{OL1S9RvWFfI+oESG3d^Rf6|hnuaC7@}sCxKw5WP4`;KC@Wne0yEup!$Yh^p-Hw> z=Digp6Af{Ty^98LihZ!`{2OIFNwDJ@CYR?T3pxE;j+QJ0ISr)^e#x4D{KqGv;T%33 zPB&{eKRy*>0b03_X3qihmb!6>nkAg)9fG}r>M{)uvZ5=8CS-cW;Fvv;vf!{kixfGu zM}H4X2yp%Sj~3uONju86#`R@h2oD*n2fW8uP!fX<3mo>O9*ZT$3sdYdVu7DcMX+f= z3LT-8KYnMS6mpdXbRL#Ails^ro_B~meQ=37#|c|yGwJAJEUg=#ii9kIW#cn*9ix!m z%LyI~{gc?qa!`v|1im=VdS;aia|L`1lb}ud%3U~QGKjd?i;9Z0!QyiFMU_Q^4&xS+ zxW%Q}z-^@wslZQ5+HE_Uua3PLhF1yDnXqRK7(Mw7z;Sg9EYp^IIOQN82oUW%x9My+2Bll8^XW;jAQKw7+IH0so2LAcQ-WxKSB_kT@j-kugR%~vTx*ZK*u*`HyoUYTyu_ICNkS7FRfV7U{Vey1lO;yx6{Y}Io` z)zbM{`M`nidh|84_CLS(VKR!GlBehg=kDII?gpVGYU=*=wq287lD}ogLpS3l2 zc_5h|kY=Uhs>o^W%=v<{vKLDnHGQ)G#Dux^gF`Wv83iHit(DU~ zZ(ktf(-MZ^z!D>+V)ESDm*1IW$ot=PhhJ3UQ(+%?#r_n#o6x)kR=?sW zJIvpuiM6jmFK`mza;2B=C7?p-J6G3$ho9Jd2UX#)=XXdyoS@6Vbr}|rB>JPTcK`Q3 z5Z3Bt<$I9{%<~aW-n2#nkSJ+jHxKIHO+NGs?>MiYl+Rxe%@#(24DQ|{c z6Yh@a;R&a)Iu=!Oi>ml;aW<1H0zp)86BWb1#hTi2SD*Q7)^SJQQABs%ozmht5&})d zgCJ5|4k71ysS6Z)g+!L;WvC14Bk5oEDIL-HJu8lyz56HVJ7hsi(DD5m6;|>i-ZS&u zpPv^UMizEeS)H0Too}ybdvS1o;JlC2G2MJ5V5NfqGt7C zP07*IwkMm+4hA4ki*gkfU*P+TqWTfQddH&;Km4%SSdO%Kh{!_K8h#u zjI>xaZl~Nl*66x4K6AP}*7D-MaGm=grW)Tpa`=U(51HL3D4?F^X{1PE@AEa%bVnqh zr1-1tddPyJ!!mav>bV+A3E;b~5}BtV?nHz4MOk@Mf`us#ki7gh$WMHgs`15zg(cX& zZ$ehP^uI&r9vet-P#!_b6KW(<%#)CWtmC^Aj&xH_CF?)nabqY3Bg``0jKOsG3SGAf z@wi(xC6B+un1T1Ny-wYoO0?64A|-F5!iTa?TmgT&sh#a%Zp(eoqCnu`c7%=FH2^y(l+o_c^qv0|fW$ghZb(b&LpU_?SdR)8T0`17SXry&vIOV z$#GE%o~He71O0TSLAb62ikho)dR`k-70ca!6Pp}WBHT1g5zbNDK+{&0)udcRiRK6w`!tVS5fLJER5@>j}wR;mu!H+3BHQ4Bg$LF&W*;nO1 ze>5`#-w<5bc@p*8LJ*E1oG+Un)cFw4kD&cQeCPzc-x`0u^D%WQ4$5%dc?j36Q2D1j znl;Q9E5KL-7d*BGWDxOkKNd)qckc-iCFVmd;{4H2Sx(;FBlt) zvezOynX1JXF?kZp-}8$jG$R4kEf~DF1rm{A$d^w=o&jtqYmIZ3DEe}E1glKd4iUWc zR&^pSDufvU&l>eH#i2tr$-(!D#G4f9{#8zBeiE(7TELnSgvK zwA~B{JxX1NllfRvIK%easT~e@1+5H z@h73T@&W%KB>{@|?3voo3lF3;>R$BxC<0z0OXfCI{jXs1g7by1-OifTWU zr^nf!OF(|{s`i4XbcCf3q!|yi z1aVQFU5i~xj9$^vDo9%RP$uUIa9rjZ+a66bEAvx6t^(-O#EK`>`Ho?Q_^E%rcpxov zaW9&lzfe7fVLy+4uJ87VBd0G)S_5%kG_zy@+elHZlY(bVY@k1)HpnV;^s#mTnkN%lixJeM(nyej8TGrzd1}Ixbb& zSf^e_4CGt$<8bG(c~|gXdUWEVgwPFWY&E8fpNN+3(no+0u(?F-k6}a@2XiwF21-G4 zo;DUka@M&-e~Q>`H!7>9?HhZapI~VpvCXLHC?uayYd4c?aPqNi}gzic?5xo{MENw}k54J>Hhh%o{ zm4=)FmE?CQ)*2kqwZ{*ts?H5ku2b?G1MK&F=Gp>p?pm2W^Q&m@KaIuAEq{b^isP(w zdl+)g6d3SWkVvdECUz0|)pgOn0(#B~a^+j8C{dztI{=0qs@Vghls5%Yr}1Bs^h)4# z%Ti&&jJwCE5e@n#DX|Uu%@@SfUW5&9y1_E0O2`RQH+RW9jg3~j3$US#dbqe3Uk@pnO~F3nufh!88pxuF{9h#=!2z==|LQaclU6Xb?c1Z^PT5cK_!3z;8}b#D zBhU(1jI+q`=)K!h8?kgo%PcaA3z8HaP97^4RrJS-u(+r+&*xE{eRGFpw;A)WyT7@7 zB9LzquzeEjv+zkex7OMnS9PUI8^pPjP=r?5>0B8DS0}+lrLUO?I0`c?*Qb2F?N%9; zaTtM9YWBx^XKjE!_ylv8wc)kA1@E}D$83_LO1vRv4Ar=?Gzy8rtv zA)Y8sjFnxCIk>=9`h%nbaS!=!t)vq%pLho2%_?bAUgU1uM$^E1ND#wfsOAYw>6#o} ze*4_Ljc&WtDgXCZ!Su|6b$r8Pbu@Q9mOXL5)8^_%Vx$Mnlu=o?o>Zyu(rg(^LX|(t`3S7 zEi_3!l{|&zo6l+{1CVo5{l@mNL*>KKQ6OUs#2ya1n)`MKZvJWC$PW)dM@Q~`yFD&)*gn1=S3F%|`^4zN zO=Ua~WG)~h{lLou7>?WsQ_P=7r+rL4)SCOzzsGuG6PEmY-N~Y{(U+O0?%hZBO#n+vY{Qlb++WY#|zxB_7=q7op z?rIswH1uDTYiq6JD$1OUD*@ZGosnPp2EZ~-N)mEVFiHEa4hEm%P%|XfUmI6N07mW0 z*P4A__<8Zc6`$k_?r;-|cZQ>7y&zHDwZEI=v5RNs5qhXj=&v1CLn7}2n&NWwULC%G zlY~9x>X9;uaxA(8UG4wWBcWMeS68>Gm=IJw+a3F3A&UEuPz4a zad%W*bB&h1{(Yp<{u#e4%PE1Cy(HVUzdJDohz$a-jz8HKVMbbHT-GQkZc?cCUhit# z0@2SU(-nZ62EZ0tvTE1zttRxF5cYnIuICzVrBFn0(@*!T!QrfMU8pRq~ne1vnaY+xvK=yjug4;HY4rvj)H)GyjimJjOe-0=sxvE zX*8a?jF=&tCs6!O=Mqr7-h^Jt3f)^@-Wwr#jm2v(A2MF+aY2tif=Cqn!KqL*x{~pC@t(O&2pSm|Kp1PWx^9-ZU$e z1`$hd6MGCX>SRMT#dVnIr1qjBCNUSTi)B> zHZS|+M$gepg$_HvVBsJiG#ykGJ9(og55hYIF;#MB@TR?k={bzTi!>m>qL6NExh$1NZDmmb-ExtRwI879r71?2gIi=CTG*$*Kqs>&zJ_z zd$~@mn+z^~yAE=7*LE|mZ6vU0e$_p=mw79=>CqXE(Rf&XA(XwU(Y>npcJLXtV4YYa z;L`Lhz&+yype#MNjaU%Ty_7>gO~Cv7D0{!-HRq0ur5Q*ax(m`iHyP0ybr_5;dbskzQEZ7((uJHP2xiG>eZV*0znS+tei?Thqkj&_D;g-2hOhly*)!sd6Fn;@ywVJ(O2P60M}2nNHvuDFlZ zMK^#ur4H-$Np6P+2rt;OUC`xCZL}im6$oQfwBeA7y@WMZODa&=%Q$xNym_Zo%hfS1 z>uLBMg+Rcpv225GV~FvBqSvgi3B9Y>oWDHL`H+3-iSMzN`~}Ywxh4PNPcl9YckXPS zz8JIrV5#Nr)#hA)tNk9>%Le$D08fY+MY1h{rKYw^tm$`SKlW311RLL!{>dh2%rGjG z>9vDX{H-87yM}kI@X95-e9=!<7E}OKEJB=0!i(RAQZ-$7V3Wg{{^Usip!`b)SOpDl z(157WHo3gM&267K;ow!w@bU4G;9oLHJ{mS*m~7ZZWD|y%Q6XPnAZqqPe1-ZXDKQ!S zwG?i)Ve#@5=u|i(seMu4j7NsnXI3BT@mObktzB0{#0^Mtj$?jN8kEy_gkfbS_#dQR z#slzg{6og8dY;NFJ1yTr*U=#$EF5>8WmZh$rN-@UNM?7Zym)wHa;1Y*@;j^}d!(Qu_ffH==~;+ZgVo8@TFSaglE@;%>U zIm-A=dqyohUnx^LPK(N^_B@B1Yjx(dG67ea`46tC$2%`GKL7St_X@(Pz6w^qWH*Rx zA32f4bG0)%+S;8Y#Z5SQ?IyH1PuR9Z5K)P6iwGmyn6h9`C>&xG)$^cJ zEde&{t)Nn?zrk9W1hOps`H|IE@UnAsc^HVV_Iki5h)eYnP@O|&@nrYf&NBe;qR)X2 zyd03aaW^Qsa`XKvWVDkz-ZcT~kpk@H5PXTu1Nz^v`rSy~%ezhDPBI+N=eD|z_Z@NT z(xKJp?r&K{*cTYfq;GTQyOYawV}%c$TDWxJMn%&Scp;_&(X9glH2%$m>&U`DKC6J` z`hJ-8bcQ>o)I%0&AxadZr-^7uY2Ui&5xeGG zd9MOQ56&Z%kQ9z>u)vJ`Nb0)@f>NZ`DS}7x8Y>~^@=EAD!zsbm9EU z#qQa{;^H9yA(jCoO_Q*08GzX_5@1hTvw`cT{HIM;#==R~BiO%ju$ReDp)u1hiftS4 z=lL)_4S0614Y%f9Ia9N{b{V1Bl*pdkSNr#vf~Daxop#`sr$*l6fHLytGO!U1 zECfbX$+S#MeoPITF~g08bi*46c5ogwydni5^%Mea{Fd05F&Cd;I&6NSiG=lP{~}x zw5$m=!?@?iGR*%n2xXFc2o?Q)YhekFcZU1Q0-*hSQEoB?DusT*Vei&S1Z4wM{7iz~ z41i4T(~bJ1s~={bIl<&RWAF+J5tZ)^fRTC)z33s@EMYg zq={5)U%q;J!*-3);bHLNC>bqp*fij&{5^XH=H;;gV>e*AU@0lLBp2y#C*ZEi8Jh7o z|AcG8LL9B2mP@z@lZ!ZqI?U@fC@YESXfmJe<(|CSH~W>Xd75%QEyH zDk`*n5NA;UQ3#uqW3=;64WVU7L8(`gV9(O%Rajk|_JgT^`?p|WHJm&3W(5RL`Wu3g-7{`dNLi2wV}iB~r)@TC`pnG+%Fe{;GNon36>0juM3l8~^lTM+JrRTeZh zY~9$Q%%?G;xw(AOp})jwf?@`WuptmWU&0%J=!8@nRmH}*__zwVn*lI;DD$;O#8tqJ z(hjO}Qf8|wU9gh-_g)1^;0dTtI#nh^dhLq}rIe@WL-HUX&3c<~?mWZdC=AFG-?U9e zzXq|UE~HUNuoADrePciA+Ax&0-$-Nu$iM>`w{)Ce1V_C~^?<5Y6Nr5yq{Z5byKYhI zN}^7pwqa)N7gO}ik?HvRQ<0<&i!CE7;eV4SSQ73a{&Ax|@2867>R%b6)jJ{4KxY9< z1(ejTD5i$*%Y9JHu~?KX<60~zmJux_v|+s&Ir3o`-P8`rNZ`u}Py`l1AH=?tdgTMl z6Di5nlsLJ&f8_}Xfno!>$%mOUqd9P&J#c<{F7)HOeJ39+fA(wn`r|{R7(AE23fANQdaE$d5Vhg6vNMv zppb7;Ct>moxf$V}<8QBUP&*+D?Hv5T-Zt@7_yJ{-RrK1kg z2Mrr&M)(6$(@BMEoZ$y$59IFZkYK814;I04_m6_K0bnx&Y14z}1;6kAh^M;wkjjYz z7qZX?|LgBZ)oWPo;J*(Gpi8kDU+piZ;=tIQ-<48Jg2?W$m zOJjn;MUhD46z_nb^_CqAyC4n5kzDggTp=Pl#GtLNa*4tf&#wCsp-PdQ<@r|-oTo#+ z;nv}Q7aC7qZ%k)dG&);-iMfTnwi5Kd4lIu0ypPIjz|;-(uLX{N|DU^3KMz_4sXMn1 zwt{+uvI!|3ktHB?g-zS7TscXKIl!?8L2q|iS=FpSB|2*YsHF)h zS=e;F3nJn)xO)W*zAxc%B}Qt{!^Z~_V58v^l@%~=>}$f@ycI*bB zCvnYAWHM#$1+_+-Y5(4_R_950SV!xab10Bi^;3-}Gq}#e){ZUWy*jmHbKfqy%%n`h z2H>64!otEY6TDIhXhh9!;SMmBBy5zo8$1RH!IjXg6Kfi6<#1m{GQ&dh5q{l$fYv=a&F$^!7M{Q1wn9dP$)6fpJo0qut53QHBpv<$2hewkwu z-qQ+TdTQ`l%?|*c7_RuPq+VOIM`(aK4&&kqI;$}=|4GpKAOY47w_ag~t!Px# literal 0 HcmV?d00001 diff --git a/docs/extraction/Extraction-Pipeline.drawio b/docs/extraction/Extraction-Pipeline.drawio index 262b42da3..743779856 100644 --- a/docs/extraction/Extraction-Pipeline.drawio +++ b/docs/extraction/Extraction-Pipeline.drawio @@ -1 +1 @@ -5V1dc5s4F/41ntl9Z+JBiA9zmThNN7tJmzZ9u9urHWITm62NXMCJ3Yv97SsBso10sDFBYJLmokZgDM85Ot866uHhfPU+dBfTWzL2Zj1dG696+LKn68hwdPofG1mnI7ZhpQOT0B9nF20H7v2fXjaoZaNLf+xFuQtjQmaxv8gPjkgQeKM4N+aGIXnOX/ZIZvlfXbgTTxq4H7kzefRPfxxP09GBqW3Hf/P8yZT/MtKyM3OXX5wNRFN3TJ53hvC7Hh6GhMTpp/lq6M0YeByX9HtXBWc3DxZ6QVzmC7pj4hm6XE9/rjztITKGn+MPZ46Z3ubJnS2zN86eNl5zCEKyDMYeu4vWwxfPUz/27hfuiJ19pkSnY9N4PqNHiH589GezIZmRMPkuxthxrq7YOAnijLqI3UZ++uyFnrww9lY7Q9nbvPfI3IvDNb0kO2sOMmQz1sJW9jLPW0JxNpru0IgTxM1YY7K58xY9+iED8Bgwje6CaSPUN3Nw6obRbxtQ3F1ATc4NGzidlsHUuwsmRoM8mDpuF8wB6i6Ypp2Xm0izWwZT6y6YWLNEMNuWmjJr3t9e04F3qzh0R7FPAnpw5y+8mR947EK9T79izehzXTyE9NOEfdI15JwhdKbbEi0oNHEe8CgOyXePQx4Qet88FbIhd+ZPAno4ooB7dPyCAe1Ta+s8OzH3x2P2MyCF8zzwEuJlxAIIZQGUwqooJUvkHI2uLyOZLh8TmpHHHj6XT57RU/fxcszesadTiDRqkfcLrvNC3wN+gJ27njOzWKT7gVlHDdwFOzNfTZgv0H9wI3/UH5PRcs5IVMNcyysBR5PIx3XcLvUcZfPMOSy2JpRnF+VffePHuA/8DtpeSCzBygAwQZDwsU1VoJRwKApA2Q9ydajaw8KSsBh+uaMD51Qcrud+REWgiA2fRIuQjLwoOqzrHtzR90kiGT8u40Sip+Nl8TUL8C0UmgiQmhCslrJ5Z0uw/u4+uZ1RUy+lQ3Z2IE91DJABKePugUSGWzdYvxkyZNNBA8w9A6IDVmVElHBFVKuhjdm0Rw1pjYreEpGDY9RQDVC1hoUMRWZlknB4c92+BsIF0B6lgSBElWkgWa+/AgVUngw8ZNiyApLNAPq82i+Up399M6Tg8bCySkjVjMCyMdC4EjL0vC+EDUOWE406ALrMoi/SQhuUO+gM6dU5ZD+2XcRCjhwMyZSEMR27o9rUnZyAZ6gXMdvpeoZc/ezi2uOpgq5qgyPocCKeIZY9ku57hsdPh9Y9Q6NEPlq5UsZOTikPJEgGTUoIo2a/cANxLVqoWShkYXm7vv90U6h4RmuqRMbJdDukedIJdvNQkyoyjvZNuNrh9rEsEaFEj6EMa13G2ovdsRtTr1G7vOi2aDyCPAeYXVlO1KiellDmn7Seq9FlCfAyYVgUPO2ASY7kGfoiLDbYdhAL2VNLvZNN2LBQRTQWM9QL0D1d3wRw+TrvmhxBhhNxTdBrdE2Onw3tuyZFqfH7UegvYokebZee2MgSymMN2ZPh9cc5i8ZRheBJVJ/oOUzgiOtARsVRFnqojsp+lGtS4wAW6sy7mk0a3GUs6s4HF4ncLmABhYWyKkBeBMgrlNy0FFGsEfxKtRx1XpklmJ6iD7L5rgRs45HrWjLKEE2UWYe67J923zw8gg7cPDT7A7T7J5EF6/3cBQCRlBmPumy0dN94PH6ygMaj3ne0nT8EkEWZLSn7q9fR9Zi+tv/oJ2JYJNBRiwleaikUMruO+oa5+4dzxpQto2yb/QMgKwugmSdgboq5hE1A7UBUcaAMlJrz+2atFlaj2QS+ILY2KGpN7zcLRfWCzP3QdhCKmos+zNpkiZggsiRJ0miCyJR9tVsSTAiQGmo/HWeWT4W3hmdh0Wu6tKrrKbdCTVHI8Gd2XnmagBXXaErOKqExvGB8zpo4MHBnbhT5ozyFvJUf/7Xz+RtDkb5YenS5ykBNDtb8IKBP/9fuwc632OH2a8nRupAymeSJyDIceXveNOPF2A0n3j6iZtPKG+daUuydVCZAMT4WejPqvj95uYeFyJj9wh3xWax0Y54agojk057fIn3v7FtbZpBv5Ag30oQbpcBIN0q4avPaL2C0Evr4dTDaoCSj8YVpJ8JphlkTp5lay5wma/HjOW3DNVtG+ZbjE5hrOIduufLbDr/CHJqsh88ezShkuxPhEttwROL2KcE3/1A1nrG5TbnpIqP1teLbquagEnHqsrJKqyKrUFOyivvuB4UVz6CdCBsagueANaMa4xmWKPWEG6lmtRLVoackrNSrxRPjNGwLHYRQZbWYvxGyGlaLJYrvOsdp5NPFAv/856cT/O3agzicv3cIb73SNU6zRQNME5y+spyGRZkmsqxqTpMzNI1xWiHXdI0bdIGICJvVuMHQW+aGEjkLkBsi+mBxORvrCLFSO2MdVHzcg+9a5AHn9ZWpV408DPI3MuxmGdCuGuJ6LQzIJdtBBuTx7xPhQCxyoGibl1aIAgeaoregmgNrjH0dxWr04M4Lffr8LCLelvyzSrLfiQXETIH9rMqWv8h+DVv+NhQQE0rN6P/unCVCkuFtluaz92PpRfHmNL9YKEWDKtBYC7s/vPUXdwK3t7vxd+6bnE9b7eV+SJgl2zQNOpxxU9BoUnfycSqEgFpbzZF5UlmOzYZiVWWIe+WzmhyIrPkyQoh0l/6IzNkN7lh/bfASVru4veLEyGiKxjVAxgFQ1KaMilxClKciw/d+OUpKOStSMU8i6Irf3Gh64T2S0Cs+f/4YA30SXkTh3Yi4VgO5bU1oqs2jzbtNK4E6LtHeqI/cUMnSQXJ/par80aegVqT3nct45epPNwz8oEAof1wwqe/OeubFo+uzF5lHk5552TEOqkNG2HmmwQbANFCd20AV00BlPHuZhpFmSGYzL1Hl18Ejqco6VI9/TX4WPPtutaA/wRhTe6Q/WdDOVnrYDyQ481JlxAqG/leqjL1ttjB0QZaATdCBNUimqoJoB/Iu9vJFZtS9kCEKDTvOLENKKnkRWdsExChvwukYKqUGSahM/SOtBg9RxKpStcRejzHnL6rMSvbKOYzZzDsRf1Fo6I+ruotIDNg27C4iTfYX362olGCMCLQ7N7XvjKhZI3OBaSvU8/V0bFlXV5bVq7eoT4WNgIX1qKYsSXgRZCO97JEmu4OMvbVfRtNl8J3ShzWt1zWkfa+lPWOHaIU1XSAWsAx+YMnEUrdFhCYnyv41bxm1kk6qV+mcemuEMq08mWx5YxQb8M0Vkkn21v7V3hhRLM3si4YvZDfp0Op7dZSRXaJkAr0p0tgWFggDbRmGATWkkDBy0vftEcYSKn7wtn3F7oxpVJYhORma2G9vijCGUAuPHFnFNCvIwG4AAk1UL+zU7b6z829g2AJIUKrDzJXpakAQVTdUBT7sclHUZIBxUQ5O68eS8BNnUcJf58ww1her7Ul+lz+89Y7rQx82vV1x7quOCaVgXdPhuXSAPcu3frTzZeGmHCWzG9VIdfceKcak/MQr7otZamLpyiYWVMBXJqd460VRsmdZtWxxbnHg/68vC9IYIfmHbTLMYMNFAemdGz2TkAUoxn4IX/w7eWBssHyY+3EaxHZrjloW7YRYmtWObuGBc7NP4/7SJt3pSBznQM1v1JkhQAc4zjvaE5kt53K/iE7ZJHURUrfy62CQqJTbDjjVvcdPMUbHGzR5FYQwtMNqky09EbBp2svAKtykpL4dkUx1rCMbvWw7zKcREwIfXEAENN7hFBU1wChU5mW3RRLLuWuEVV7oceMGk34dArXF9ppHkGLTLlIiRcNtToH+mmBO9LVT4iwfocJlG5+q8wZqbpRTh9biKSTQzQbksqqepqjMlroN7QR+gHpH7ETQ+n7gwE6dB5yqrUUcJMqwYtVXSBYUJwYGOniFLl/R78uao+YajhdTmWeKSs4cXZlYKbHG8qiZk1cDdO6ktiysC1TUOwuB0oENzSMdmkjqdh4GNphCEsyd8haPT2YbjpiO04AYdqOpBVxciR4t3KAHRV8T8M6iFD0Wfw1IOHdncgT21o0pHeaELRnQPhDWVHKUtMWNdiRh+jN1FA62Pe/MARI7nfMqoIMTTx2Ba1jXW1SkpvXtV7GEbjMLTqQoTVzpYFTt6oMG+RuZDXf1QcDmptva5gisTGPGzXUwSdcvgeeo6TMJ0yUUwPkrEn72nnzvGT49JKwmLgkZd1nHY6F+wDAgJW8DGQh18WFwBwHBWL6ep8ZxtGTWa+peFTBBifVK19G7bSn6kZXoHSCyKQaOZQo7QO1bHRXP71efPv84i7yrmy+DwRqNyaf1kJf4FnWGDL1FshGrAHnbG5DoQiIFm0DHUgBGPsNqh7FjjaqSI7G+vF6tvI/bTkQnY7F5ldgau2qjFyymeerTyfsa7ez1dqdk/rAsEakvmIoAYxTHe7EhGM3UaIEqroD1uTx2UP+qraPX5yaL6lyqrMZVQz7s2589N4I2DWHn72M3XrJacLa2T3Fsp4LrIy7WLNdfvg5VBXJ5HR3aGu8TIoi6avIVAZ0bigVBW+LUEjwTKdldugklFqNddYlTehgSFs/YXs4MlFsy9tgV/wE= \ No newline at end of file +7V1bc6M4Fv41qZrdqrhA4vrYSTqzmel0J53unsvLFrGJQ9o2HowTux/2t68EyAbpYMCWwCQ9XbUbCyzwOZ/OXUcn+Hy6+jXy5o/X4cifnCBttDrBFycIuY5O/pcOrNMB07LTgXEUjNIhfTtwF/zws0EtG10GI39RuDEOw0kczIuDw3A284dxYcyLovCleNtDOCk+de6NfWHgbuhNxNE/glH8mI46yN6O/8cPxo/sybrlplemHrs5+yWLR28UvuSG8PsTfB6FYZz+NV2d+xNKO0aX9HuXJVc3Lxb5s7jOF27XXx5Xlv/976ev/tpd3HxbPixPsZNO8+xNltkvzt42XjMSROFyNvLpLNoJPnt5DGL/bu4N6dUXwnMy9hhPJ+STTv58CCaT83ASRsl3Mcaue3lJx8NZnHFXp9Nkj/Wj2F+V/iB9QyYCLz+c+nG0JrdkX7A0PDDTL2XgMnBG/JctqxiQHnNcYizxMnCMN3Nv6Uf+yEjYhJx2f8lpMCxkxMS6xcjbAjmRa+KJfrF+/LHytfuFcf45/njqmp2SU6AdQOFydFo2j04Ld0xO3F9y2qZeICYy3c7RifpLTqw7HDoxQl0TlNkKfSSoaWkFcuqa3Tk5tf6SE2vWsZFTBOfd9RUZeL+KI28YB+GMfLgJ5v4kmPn0RjTAA/Iy1oS82dl9RP4a07+QhrRTzT3VdIEbhDhxkeSLOAq/+4zos5DMXORDNuRNgvGMfBwSkvtk/IySOiBm7LvswjQYjehjQB4XUXAI+5iVL7LKBniFVfFKlMwFLl1dLES+fEq4Fj6c4HfixVNy6S5ejuhvPEGERBrxdAYl9/lR4AMPoNeuptTf4Plese6I5zCnV6arMfWxBvfeIhgORuFwOaUskrDaipafqwnsY7ouzz1X2UpzqwXXmGB2Xv+nbxxE757NoO0kiWXzzoVuiWTRIQlkm6roUsMcLqHLbjrvT63uaGEJtDj/ckMG3hGJuJ4GCyIFedqwdTSPwqG/WFQrvHtv+H2cCMdPyzgR6+l4XfqaJfQtlZs6IDghslrKlp7owP7mPXu90VSH8iG76ohLHQNs0JWhWwzLXHuz9ZthQ7YcNMDmMyA+YFV2RA2PRLUm0jnPAtDOutaq6K0RRmiihiSQqjNaiKTIDM0wOv9w1b0GwiWkbaSBIIoq00CiXn8FCqg+G1i4q2MFJJoB5H21Xwim//VmWMEijXWVkKoVUSdHo1oJGcjgo9kGhiIyrToBqEa6pYkm2lC6hw4R2h8lu2nbR1qIAYTz8DGMYjJ2QzSqNz4C7xCVge14vUOmgvJ0PUG43xqhAR+OxDvEolfSf++w+XLo3Ds0jCNQzKzkISOKI5DEaVNCGJJ9ww2JpWihdkkhCsvr9d3th1LFM1wTJTJKlluV5kkX2Id7SarIaOyfMLXDbGRRIloArQ1ltEYirf3YG3kx8Ry1i7N+i8YG7KkAu7LkqLF/akKmj1IAJRQoa9cMrZGAbyQMywKoPTDJdXGFHkSLDW17SAvRU0u9k03osFRFtBY3RCXUPV7fBHD5eu+aNGDDkbgm+mt0TZqvhu5dk7L0+N0wCuaxwI+uK1BM2+TLLVjJSd6mNEUSIksVCY+gCsV26oZdHZEyrrL4w/6U2U1pSbocoIU6G0+yXYP7TAvZieEyudsHWkCxoawikBUEslIlLy1L5OsFvxFVRzxYag6ml8iLbL4rELb18LWU1DLEE2UmIhKd1P7biA34wGxEc+Do+X8CWzAaFG4AmKTMgkSi5dJ/C7L5YgEtSDRwtdw/HWCLMoNSdFqvFlcj8rODhyARwzyDGm0tONRSKAU70geGmf+HC+aULVKZWKIVRFYWRTOPwOTktwGaJmRwQsFFRxlZJKf5Tak2VqtJBVNySNGUmuVvlxT712buJm0PSSG59sOUJk34PJElSJJW80Sm6K1dh7NxCGSIus/KmfUz4p3Rs7T+Nd1o1ffMW6mmKAX8qV3Mx5s1lacym8KqoTH82egd7ZVBiTvxFotgWOSQvwriP3N//0WpSH5Y+ulilRE1+bBmH2bk7f/Mf8h9i37cfi35tC7lDNsnGS6job/jl2ZYjL1o7O9iaras/FGh88fORWUCHGNjkT8hDvyzX3hZiI3ZE27CgIZMNwaqwYlItuzZFOnvzr61BYM4kctNpHETpYQRJkpQtfnZBwCthj5+HUBzagLNxkeFNMOUhDRT6xhpNeKMdZGm7YM0vTWk2TWhxipejwVqusbnLbbNeprCzbD4ybApTKYacjVK/SohtxFUW8T8VQAMDJ8NVDfw/CsHYhiq6oXbkSEO28XiH6TvLdyKE+lWy8JNjH+2hrRS1PQNDYjvxoL3lT2IU3U8rFSjoUZEEETDgrxYXE8DNhAr0oFVKY6Yddw3qx4XpYiJ9rXqneJEht0uAO193cfXAkAm2SoByGJLR4JAjPnSEUu0muqiEAt1KJZoz6lGogz/snVfkRViVcKHNTQ5EvhYfFhCs/eDDt8YzeKDkaphgwDYcIUY5P+9KQ0SJsPbCOZn/5+lv4g3l9nNXKEGVJ9Bmz397q+/eGO4EdSHIDdvcj1tSlV4EIfubQhTr45GK2jKhlgylFlVeqabCs0CXBGTyuLPNhQJqMPcy4BmrCG2FotsINZdBMNwSie4oS1+wVs+LeP5Mt7ec2SMNE2h2hIoznGAsg9lnHSgFNZOTlLqnoeTiZ8s1avZQ7gvQ8k6/ZY8Frz6fjUnjyDcQtoDeWRJYzfhZT+Gs1M/BRtNlv27VhFX18AwEA+MsragQCWuqaokyIU0/05sZIL7QFCUCm8GmHPCLrGWumsmYl3nmcj2AFSxUNny1jUJ1htPq72yBeTDjR8F5GfRHGJHGQRU0yg8rgSCXbTk8L4hNp0PqrQcYtM10SZ8vyJSggIRaP5pat8pU7O2nhxo98hnnyBsWZeXlnUiN6mtwlLgu2CaojJgRQCtdHbVNdHko/DWfhk+LmffCX9oC1ek6dp3KZ2KesQrrCGOWcBuMMcSmaWuZbImBrP/Z15TbiVNxS7TNfXWGMW7xACbbMD2Vsgm0fhO2PSmuGK5Lrd8kA2dgIEBcaeQNWIC4O2xxuaCUYYJlVWhVteMLgbGEzvhTTHGcPnUPHKhNYMAX0Mda8C9VxxfVJfRI3vg5v5zDLtAJt2FQmfmQMv/B209MFQ52TZUZC862ckARVKBnNY/y5BdOF0kGHtHjTA0X20vslmI95wzs8nLptOVx1JlLCoFNaTV66kCnvW77dgDM48LZnHnDYZWtZLsnZ7lNKm/8MpbEdVaWEjZwoIKOurEqK/9xSI5LWK/7EOhEPvr1QUcyrqJwid6bh4lGy4LgOYmegkj6gyPggi++bfwnsJgeT8N4jRo6kmOkJWdQlMbao03TOLC6tPs4gYpnR2jkD8uA9pqrM4UAZpuMOxoz+FkORV35/XKLpHFSGQV5ICu80q56+CG7Nbq5TRqbtAUVZDOjjrsrg09cFbFYcQq7Q0trxG9qQ46otFLDyJ6HlIh8NEDREDrTaX0ss2Gpcq8bjd6vrxPIlnFngEfvNl4IEOgdtjRqAErmA8OtAlot7MU0NIIzL+9dk6cFrewQ9uzW+1Pv0nnHJPWYukK0M0G5LKrDLY1KnhbOoWxgnsNmr92fhYjcEBShVO1tYhniTLcryqAuEtzQidKDL3yDiTeMRiImkNyvcDBXGatMWquHKRMrMg+/bmoBsjaSW1ZWBeoqJ/jNiU44JmmCLJvMOKrbeXRGejqrwuE7pW/2Dx1ym2NxBoYxW41wQAc+7Itd1uUnlJ6NRunJavgNSKdxhE19sHrl2H02X8O/Bf48nlISySSqE6flyFmyp7ZCDZ0lLgNBAnVhXDAloqcPsvOjtUWS6pgUguoBAQ1SlSvFu+31YkNixN7wGSTj+2IHHaBUggZBXC/rm4//3O68C8/fHGctT4Kb9fnVScSR/48OZ6GI3nXbVmRzR8TD6deLYCUloRWZCApe7b7O/nElxyW8mav/Ui7EHcktYNY3BG+FbuNtyMJO8INJEwmr4gwvD2b4x9PP9zZfz3biaPpr24IHAcvCszHcHq/rBFYk7BSba7vDDbEHCGDTsF+scs5WXeVLp7iz79fPX394+XpYTo9C7H2xQDpkyqZxdybFQhVlj3WoOwxmyVgA5kc9UfZng5O193FXrwkum746M3GeUUX8DPSerTk1WpnnYklFfzIQhF0VXvLOFxkeqyR0TunKE04YJ6dmBewlgQN7qqgU2MgOVoxvYURZAq7gKSQoTtBKJUnUGsXItSFUoqST9Ht0l/6adR8J2CKpQuH+0ytBCKbl8Zx5VesXrsi4ijDNwIBUR78kQwIcY9RTUHSEBf1+c2LHFFycEJIAvsxK1batMQAAxetQgAov1OEgWwvUe8YH4Vx2lgcXximpAAWy8GwECFUa9MqCkS3Si0K3qoIQBbvCG46oVSd+KOO+eWbf5WrgcQ+eEsAMHS+Oh7btcPX6iBQfsqCmvXfL74rUAHYLNoCgApQ1d4URgCUJFLkHGw6B9CDOwpIAKsxe4MNWV6Cw0dlNm23uhMQrfmOOQOhX0JChlDQeesA2+AxVaqMw9v1l8eV5X//++mrv3YXN9+WD8s6EbpcVUrtWhOAVjuCKVzK1wIsZlVnT4JEgQxm6UQpZ8fuislWKSFaj6mZNw/8qFyuC2u5rhhxICnySzibUCI80EOeyOvmj1PR/E2CiO6QrdotE5RKE8l1ngAmKnmPYN53cHYo+HqiFXn050JJ5MJGVglcUFXgCb6daMgd/blP8pdCmweHgq8no/ts630duRxnZWIV1k91zw9IbcquEqmuhYqWlsYdgFq7C4umFWfaZPHlZ1BhgrcWvMu5bsmRZa/Sd6sQRhXpY7dDz23ncpSYYD+MQjbfVcsVFWarGfbykjF1y6d3sW/5YQ4b82EOKA2iKhIOr5UaldLHd/RKU629swNoXmvvkCddaW1dw1wkgK+WqK+2neJMLjdRidYmzPfWuduylVP6xhbX7U3PzkXY4jSdUa5J8NP8rARyxzjmKuV1/tTP2jjW+dgYv61asfmJasQH+4+1nb3G81grL3LsCmtC1ydtT6jZ3MlomN+FrRppkKPz6pC2K8h65EDjVJ3Bb5WtCzSH2+GF+Yilao+6Xg8nsdVMWjyblsqe7Lk1Mu15nuCkbM/J5gE7rmlTtlGTd/YOahpDP2dzle2EbOQN8scjWHCbbBsArVGOz8NSO63Vw3z+Pb29cGsRZt6M7msBscN/5Wf8hebOuaMVWEOs/I60Nl1KsIeeEjCxgd3bAl6yH00nnoXR1JuIU5fDkivtTxJt8p5b+bzXA/W8HC3vldggQc7viW4R9ztP3NxThX7zo+CBUD7d1rdnjwGP7s+9/MOLZsGs5ASCT3P6BAJG8+zBC+ivmi7GlIE7VPNxHkXC2eZ0f6ENaFIoFWmZ/J4veeJPZiDiZI+I2l5HzEpxDtkKrD5xz+7UaOe9Q2fPPJgpHJRmOI66zYQw3MqljppYfl5a+aM+R/Vzha2HWfR60QIzMWjRQ4hWZ4S1ttEJhMUrSpPKQomJ+ErHEpyo2gwF46RGj6n+B5h2bcOuPqDY6FJZubyVs2+ESdeMbkNMQLOgn1CDOjR0BTVHK7YrRkW4OFqxBe5+MORPBxVC86pRWN5doP1Q1yx8pgpz/TPcVV+Nuvy5UtiENgyo2koEgwodEah+QurASBK8PbHVGCpuLYZaMN8/hokBv35F5rsssePwu5SAyq1WLXfc9vbFnIf3EyDVQgSo8QRDkeoAApygp9Ug+5F15zxs2drCqcQuZC1I6s9JPkYhXUxbi5V217sORz694/8= \ No newline at end of file From ac2e67dd3f214a6fe34b8a649cc134b2d8b1e8d8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 12:15:57 +0100 Subject: [PATCH 077/138] Rename ExtractedFileStatus Unused -> None --- .../Smi.Common/Messages/Extraction/ExtractedFileStatus.cs | 2 +- .../Execution/ExtractJobStorage/ExtractJobStore.cs | 4 ++-- .../MongoDB/ObjectModel/MongoFileStatusDoc.cs | 2 +- .../java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java | 2 +- .../Execution/ExtractJobStorage/ExtractJobStoreTest.cs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs index da87ca49a..0cbab544d 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs @@ -5,7 +5,7 @@ public enum ExtractedFileStatus /// /// Unused placeholder value /// - Unused = 0, + None = 0, /// /// The file has been anonymised successfully diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index 38011f4ab..127c657ae 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -43,8 +43,8 @@ public void PersistMessageToStore( [NotNull] ExtractedFileStatusMessage message, [NotNull] IMessageHeader header) { - if (message.Status == ExtractedFileStatus.Unused) - throw new ApplicationException("ExtractedFileStatus was the default unused value"); + if (message.Status == ExtractedFileStatus.None) + throw new ApplicationException("ExtractedFileStatus was None"); if (message.Status == ExtractedFileStatus.Anonymised) throw new ApplicationException("Received an anonymisation successful message from the failure queue"); diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs index 19d0299ec..917b580df 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs @@ -63,7 +63,7 @@ public MongoFileStatusDoc( OutputFileName = outputFileName; WasAnonymised = wasAnonymised; IsIdentifiable = isIdentifiable; - ExtractedFileStatus = (extractedFileStatus != ExtractedFileStatus.Unused) ? extractedFileStatus : throw new ArgumentException(nameof(extractedFileStatus)); + ExtractedFileStatus = (extractedFileStatus != ExtractedFileStatus.None) ? extractedFileStatus : throw new ArgumentException(nameof(extractedFileStatus)); StatusMessage = statusMessage; if (!IsIdentifiable && string.IsNullOrWhiteSpace(statusMessage)) throw new ArgumentNullException(nameof(statusMessage)); diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java index 2c4bb15ad..bbe3a6353 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java @@ -5,7 +5,7 @@ public enum ExtractedFileStatus { /** * Unused placeholder value */ - Unused, + None, /** * The file has been anonymised successfully diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs index 1d403abcf..5b7a5e78e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs @@ -81,7 +81,7 @@ public void TestPersistMessageToStore_ExtractFileStatusMessage() var message = new ExtractedFileStatusMessage(); var header = new MessageHeader(); - message.Status = ExtractedFileStatus.Unused; + message.Status = ExtractedFileStatus.None; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); message.Status = ExtractedFileStatus.Anonymised; From 0211eae3961446fe64d1891ed17d1368e6103194 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 12:19:18 +0100 Subject: [PATCH 078/138] Clarify test method purpose --- .../ExtractionRequestQueueConsumerTest.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs index 4656bd4c9..c778f0a9d 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -1,5 +1,4 @@ using Microservices.CohortExtractor.Audit; -using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.ProjectPathResolvers; using Microservices.CohortExtractor.Execution.RequestFulfillers; using Microservices.CohortExtractor.Messaging; @@ -56,7 +55,7 @@ public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() GlobalOptions globals = GlobalOptions.Load(); globals.CohortExtractorOptions.ExtractAnonRoutingKey = "anon"; globals.CohortExtractorOptions.ExtractIdentRoutingKey = ""; - TestRoutingKeys(globals, false, "anon"); + AssertMessagePublishedWithSpecifiedKey(globals, false, "anon"); } [Test] @@ -65,13 +64,19 @@ public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() GlobalOptions globals = GlobalOptions.Load(); globals.CohortExtractorOptions.ExtractAnonRoutingKey = ""; globals.CohortExtractorOptions.ExtractIdentRoutingKey = "ident"; - TestRoutingKeys(globals, true, "ident"); + AssertMessagePublishedWithSpecifiedKey(globals, true, "ident"); } - private static void TestRoutingKeys(GlobalOptions globals, bool isIdentifiableExtraction, string expectedRoutingKey) + /// + /// Checks that ExtractionRequestQueueConsumer publishes messages correctly according to the input message isIdentifiableExtraction value + /// + /// + /// + /// + private static void AssertMessagePublishedWithSpecifiedKey(GlobalOptions globals, bool isIdentifiableExtraction, string expectedRoutingKey) { var fakeFulfiller = new FakeFulfiller(); - + var mockFileMessageProducerModel = new Mock(MockBehavior.Strict); string fileMessageRoutingKey = null; mockFileMessageProducerModel From 23d7f4de0def70bd5523cb89454185c443793785 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 12:23:26 +0100 Subject: [PATCH 079/138] Use TestTimelineAwaiter instead of sleep --- .../Execution/FileCopierHostTest.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs index 4cf0a77c2..686addfbc 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -94,8 +94,7 @@ public void Test_FileCopierHost_HappyPath() ExtractedFileStatusMessage statusMessage = null; consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(ea.Body.ToArray())); model.BasicConsume(outputQueueName, true, "", consumer); - Thread.Sleep(500); // TODO Race-y - Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status); + new TestTimelineAwaiter().Await(() => statusMessage.Status == ExtractedFileStatus.Copied); } #endregion From 470719989ca6276de9426382890a9bd98eb85760 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 12:32:59 +0100 Subject: [PATCH 080/138] Update Fatal logging in Consumer and refactor sleep out from tests --- .../Smi.Common/Execution/MicroserviceHost.cs | 11 +++--- src/common/Smi.Common/Messaging/Consumer.cs | 3 +- .../Messaging/FileCopyQueueConsumerTest.cs | 34 +++---------------- 3 files changed, 10 insertions(+), 38 deletions(-) diff --git a/src/common/Smi.Common/Execution/MicroserviceHost.cs b/src/common/Smi.Common/Execution/MicroserviceHost.cs index 98c195483..7f0c7ef5b 100644 --- a/src/common/Smi.Common/Execution/MicroserviceHost.cs +++ b/src/common/Smi.Common/Execution/MicroserviceHost.cs @@ -1,10 +1,5 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Reflection; using DicomTypeTranslation; -using DicomTypeTranslation.Helpers; using JetBrains.Annotations; using NLog; using RabbitMQ.Client; @@ -13,6 +8,10 @@ using Smi.Common.Messages; using Smi.Common.Messaging; using Smi.Common.Options; +using System; +using System.Diagnostics; +using System.IO; +using System.Reflection; namespace Smi.Common.Execution { @@ -205,8 +204,6 @@ public virtual void Stop(string reason) /// public void Fatal(string msg, Exception exception) { - Logger.Fatal(exception, msg); - if (_stopCalled) return; diff --git a/src/common/Smi.Common/Messaging/Consumer.cs b/src/common/Smi.Common/Messaging/Consumer.cs index 3ebc4472b..274a6a3da 100644 --- a/src/common/Smi.Common/Messaging/Consumer.cs +++ b/src/common/Smi.Common/Messaging/Consumer.cs @@ -213,6 +213,8 @@ protected void Fatal(string msg, Exception exception) _exiting = true; + Logger.Fatal(exception, msg); + ConsumerFatalHandler onFatal = OnFatal; if (onFatal != null) @@ -221,7 +223,6 @@ protected void Fatal(string msg, Exception exception) } else { - Logger.Fatal(exception, msg); throw new Exception("No handlers when attempting to raise OnFatal for this exception", exception); } } diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs index 7bee3acca..d91798d75 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -6,14 +6,12 @@ using RabbitMQ.Client; using RabbitMQ.Client.Events; using RabbitMQ.Client.Framing; -using Smi.Common.Events; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; using System; using System.Collections.Generic; using System.Text; -using System.Threading; namespace Microservices.FileCopier.Tests.Messaging @@ -91,20 +89,9 @@ public void Test_FileCopyQueueConsumer_ValidMessage_IsAcked() var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); consumer.SetModel(_mockModel.Object); - var fatalCalled = false; - FatalErrorEventArgs fatalErrorEventArgs = null; - consumer.OnFatal += (sender, args) => - { - fatalCalled = true; - fatalErrorEventArgs = args; - }; - consumer.ProcessMessage(mockDeliverArgs); - Thread.Sleep(500); // Fatal is race-y - Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); - Assert.AreEqual(1, consumer.AckCount); - Assert.AreEqual(0, consumer.NackCount); + new TestTimelineAwaiter().Await(() => consumer.AckCount == 1 && consumer.NackCount == 0); } [Test] @@ -118,20 +105,9 @@ public void Test_FileCopyQueueConsumer_ApplicationException_IsNacked() var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); consumer.SetModel(_mockModel.Object); - var fatalCalled = false; - FatalErrorEventArgs fatalErrorEventArgs = null; - consumer.OnFatal += (sender, args) => - { - fatalCalled = true; - fatalErrorEventArgs = args; - }; - consumer.ProcessMessage(mockDeliverArgs); - Thread.Sleep(500); // Fatal is race-y - Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); - Assert.AreEqual(0, consumer.AckCount); - Assert.AreEqual(1, consumer.NackCount); + new TestTimelineAwaiter().Await(() => consumer.AckCount == 0 && consumer.NackCount == 1); } [Test] @@ -150,8 +126,7 @@ public void Test_FileCopyQueueConsumer_UnknownException_CallsFatalCallback() consumer.ProcessMessage(mockDeliverArgs); - Thread.Sleep(500); // Fatal is race-y - Assert.True(fatalCalled, "Expected Fatal to be called"); + new TestTimelineAwaiter().Await(() => fatalCalled, "Expected Fatal to be called"); Assert.AreEqual(0, consumer.AckCount); Assert.AreEqual(0, consumer.NackCount); } @@ -173,8 +148,7 @@ public void Test_FileCopyQueueConsumer_AnonExtraction_ThrowsException() consumer.ProcessMessage(mockDeliverArgs); - Thread.Sleep(500); // Fatal is race-y - Assert.True(fatalCalled, "Expected Fatal to be called"); + new TestTimelineAwaiter().Await(() => fatalCalled, "Expected Fatal to be called"); Assert.AreEqual(0, consumer.AckCount); Assert.AreEqual(0, consumer.NackCount); } From 52a2ac96eb87022b229c4cbab1a2861a5ba5940f Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 13:39:39 +0100 Subject: [PATCH 081/138] Fix FileCopierHostTest --- .../Execution/FileCopierHostTest.cs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs index 686addfbc..0049a1044 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -1,4 +1,4 @@ -using Microservices.FileCopier.Execution; +using Microservices.FileCopier.Execution; using Newtonsoft.Json; using NUnit.Framework; using RabbitMQ.Client; @@ -10,7 +10,6 @@ using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Text; -using System.Threading; namespace Microservices.FileCopier.Tests.Execution @@ -64,6 +63,7 @@ public void Test_FileCopierHost_HappyPath() mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject); var host = new FileCopierHost(globals, mockFileSystem, false); + tester.StopOnDispose.Add(host); host.Start(); var message = new ExtractFileMessage @@ -80,21 +80,13 @@ public void Test_FileCopierHost_HappyPath() using IConnection conn = tester.Factory.CreateConnection(); using IModel model = conn.CreateModel(); - - var timeout = 5; - while (--timeout >= 0 && model.MessageCount(outputQueueName) == 0) - { - Thread.Sleep(1000); - } - Assert.True(timeout >= 0); - - host.Stop("test finished"); - var consumer = new EventingBasicConsumer(model); ExtractedFileStatusMessage statusMessage = null; consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(ea.Body.ToArray())); model.BasicConsume(outputQueueName, true, "", consumer); - new TestTimelineAwaiter().Await(() => statusMessage.Status == ExtractedFileStatus.Copied); + + new TestTimelineAwaiter().Await(() => statusMessage != null); + Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status); } #endregion From 8d07a647ae9ad3b877cc9dafc351f5d0c18aa0e8 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 13:52:11 +0100 Subject: [PATCH 082/138] Keep LGTM happy --- .../Messages/Extraction/ExtractMessage.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs index 675449a6f..71e6129e1 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs @@ -76,14 +76,12 @@ public bool Equals(ExtractMessage other) if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return true - && ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) - && string.Equals(ProjectNumber, other.ProjectNumber) - && string.Equals(ExtractionDirectory, other.ExtractionDirectory) - && JobSubmittedAt.Equals(other.JobSubmittedAt) - && IsIdentifiableExtraction == other.IsIdentifiableExtraction - && IsNoFilterExtraction == other.IsNoFilterExtraction - && true; + return ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) + && string.Equals(ProjectNumber, other.ProjectNumber) + && string.Equals(ExtractionDirectory, other.ExtractionDirectory) + && JobSubmittedAt.Equals(other.JobSubmittedAt) + && IsIdentifiableExtraction == other.IsIdentifiableExtraction + && IsNoFilterExtraction == other.IsNoFilterExtraction; } public override bool Equals(object obj) From c23bcac31ae277ac84651f1dd337cbfce5b6a827 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Tue, 1 Sep 2020 14:15:49 +0100 Subject: [PATCH 083/138] Fixup java tests --- .../test/execution/ExtractImagesTest.java | 6 +++++- .../fileUtils/ExtractImagesCsvHandlerTest.java | 16 ++++++++-------- .../ExtractedFileVerificationMessage.cs | 16 +--------------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java index 6b640ac8b..be41bdf77 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java @@ -90,7 +90,8 @@ public void testExtractImagesFromSingleFile() throws Exception { "MyProjectID", "MyProjectFolder", null, - false, + false, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -154,6 +155,7 @@ public void testExtractImagesFromMultipleFiles() throws Exception { "MyProjectFolder", null, false, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -217,6 +219,7 @@ public void testMissingFile() throws Exception { "MyProjectFolder", null, false, + false, null, null); @@ -246,6 +249,7 @@ public void testEmptyFile() throws Exception { "MyProjectFolder", null, false, + false, null, null); diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java index 7b2f14501..5e3a09366 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java @@ -27,7 +27,7 @@ public void testProcessingSingleFile() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, false, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, false, false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -83,7 +83,7 @@ public void testProcessingSingleFileWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -138,7 +138,7 @@ public void testProcessingMultipleFilesWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -206,7 +206,7 @@ public void testIdentifierSplit() throws LineProcessingException { UUID extractionUid = UUID.randomUUID(); ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(extractionUid, "MyProjectID", - "MyProjectFolder", null,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); + "MyProjectFolder", null,false, false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -251,14 +251,14 @@ public void testModalityRequirement() throws LineProcessingException { boolean thrown = false; try { - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa",false, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); } catch (IllegalArgumentException e) { thrown = true; } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); thrown = false; @@ -269,7 +269,7 @@ public void testModalityRequirement() throws LineProcessingException { } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); @@ -283,7 +283,7 @@ public void testModalityRequirement() throws LineProcessingException { // Happy path - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); handler.processLine(1, new String[] { "s1" }); diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs index e3e38a2ec..645e3f61c 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs @@ -26,22 +26,8 @@ public class ExtractedFileVerificationMessage : ExtractMessage, IFileReferenceMe [JsonConstructor] public ExtractedFileVerificationMessage() { } - public ExtractedFileVerificationMessage(Guid extractionJobIdentifier, string projectNumber, string extractionDirectory, DateTime jobSubmittedAt) - : this() - { - ExtractionJobIdentifier = extractionJobIdentifier; - ProjectNumber = projectNumber; - ExtractionDirectory = extractionDirectory; - JobSubmittedAt = jobSubmittedAt; - } - - /// - /// Creates a new instance copying all values from the given origin message - /// - /// public ExtractedFileVerificationMessage(ExtractedFileStatusMessage request) - : this(request.ExtractionJobIdentifier, request.ProjectNumber, request.ExtractionDirectory, - request.JobSubmittedAt) + : base(request) { DicomFilePath = request.DicomFilePath; OutputFilePath = request.OutputFilePath; From e4bb0aca60296c3d06fbd5517f18abbd6652e998 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 05:01:38 +0000 Subject: [PATCH 084/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.7 to 3.5.9. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.7...v3.5.9) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 294d4bba7..325eca381 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.7 + 3.5.9 From f64f22929137c0fa034d93767690241bf8991545 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2020 09:44:20 +0100 Subject: [PATCH 085/138] Bump HIC.RDMP.Dicom from 2.1.9 to 2.1.10 Bump HIC.RDMP.Dicom from 2.1.9 to 2.1.10 --- PACKAGES.md | 2 +- .../Microservices.DicomRelationalMapper.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index 015f9e442..aa85131b7 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -14,7 +14,7 @@ | fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | | HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.9](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.9) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.10](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.10) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | | HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index 33d39261f..7337a98b8 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -18,7 +18,7 @@ - + From eb008e1504be54678f9a77fcbc1d69cc68ee51df Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 3 Sep 2020 11:06:22 +0100 Subject: [PATCH 086/138] Class comments for reviewer --- .../IsIdentifiableReviewer.csproj | 1 + .../IsIdentifiableReviewerOptions.cs | 5 +++ .../IsIdentifiableReviewer/MainWindow.cs | 28 ++++++++++++++++ .../Out/IRulePatternFactory.cs | 3 ++ .../Out/IgnoreRuleGenerator.cs | 17 ++++++++++ .../Out/MatchProblemValuesPatternFactory.cs | 10 ++++++ .../Out/MatchWholeStringRulePatternFactory.cs | 9 ++++++ .../IsIdentifiableReviewer/Out/OutBase.cs | 23 +++++++++++++ .../Out/OutBaseHistory.cs | 15 +++++++++ .../IsIdentifiableReviewer/Out/RowUpdater.cs | 21 ++++++++++-- .../Out/SymbolsRulesFactory.cs | 6 ++++ .../Out/UpdateStrategies/IUpdateStrategy.cs | 11 +++++++ .../ProblemValuesUpdateStrategy.cs | 8 +++++ .../UpdateStrategies/RegexUpdateStrategy.cs | 8 +++++ .../Out/UpdateStrategies/UpdateStrategy.cs | 11 +++++++ .../IsIdentifiableReviewer/Target.cs | 23 +++++++++++++ .../UnattendedReviewer.cs | 32 +++++++++++++++++++ 17 files changed, 229 insertions(+), 2 deletions(-) diff --git a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj index 08babf085..3c9a85ffe 100644 --- a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj +++ b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj @@ -10,6 +10,7 @@ 8.0 full true + true diff --git a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs index 1432379de..f258cb528 100644 --- a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs +++ b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs @@ -3,6 +3,10 @@ namespace IsIdentifiableReviewer { +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + /// + /// CLI options for the reviewer + /// public class IsIdentifiableReviewerOptions { @@ -47,4 +51,5 @@ public class IsIdentifiableReviewerOptions )] public bool OnlyRules { get; set; } } +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member } \ No newline at end of file diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 9e9010453..419eac03f 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -15,16 +15,41 @@ class MainWindow : View,IRulePatternFactory { private readonly List _targets; + /// + /// The currently selected database which will be updated when performing redactions (when not operating in Rules Only mode) + /// public Target CurrentTarget { get; set; } + + /// + /// The report CSV file that is currently open + /// public ReportReader CurrentReport { get; set; } + /// + /// Generates suggested ignore rules for false positives + /// public IgnoreRuleGenerator Ignorer { get; } + /// + /// Updates the database to perform redactions (when not operating in Rules Only mode) + /// public RowUpdater Updater { get; } + /// + /// Width of modal popup dialogues + /// public int DlgWidth = 78; + + /// + /// Height of modal popup dialogues + /// public int DlgHeight = 18; + + /// + /// Border boundary of modal popup dialogues + /// public int DlgBoundary = 2; + private ValuePane _valuePane; private Label _info; private TextField _gotoTextField; @@ -34,6 +59,9 @@ class MainWindow : View,IRulePatternFactory private Label _updateRuleLabel; private CheckBox _cbRulesOnly; + /// + /// Record of new rules added (e.g. Ignore with pattern X) along with the index of the failure. This allows undoing user decisions + /// Stack History = new Stack(); ColorScheme _greyOnBlack = new ColorScheme() diff --git a/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs index 912510260..afebf0f91 100644 --- a/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs @@ -2,6 +2,9 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Interface for classes which generate patterns for classifying as true or false positives and describing which part to redact. Can be a strategy (e.g. use whole of the input string) or involve user input (e.g. get user to type in the pattern they want). + /// public interface IRulePatternFactory { /// diff --git a/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs b/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs index cd46f8174..4db1e9d0a 100644 --- a/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs +++ b/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs @@ -5,14 +5,31 @@ namespace IsIdentifiableReviewer.Out { + /// + /// + /// Implementation of OutBase for . Base class should + /// be interpreted as rules for detecting which are false positives. + /// + /// See also: + /// public class IgnoreRuleGenerator: OutBase { + /// + /// Default name for the false positive detection rules (for ignoring failures). This file will be appended to as new rules are added. + /// public const string DefaultFileName = "NewRules.yaml"; + /// + /// Creates a new instance which stores rules in the (which will also have existing rules loaded from) + /// public IgnoreRuleGenerator(FileInfo rulesFile):base(rulesFile) { } + + /// + /// Creates a new instance which stores rules in the + /// public IgnoreRuleGenerator() : this(new FileInfo(DefaultFileName)) { } diff --git a/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs index 0197478ce..0b82828f2 100644 --- a/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs @@ -1,14 +1,24 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using Microservices.IsIdentifiable.Failures; using Microservices.IsIdentifiable.Reporting; namespace IsIdentifiableReviewer.Out { + /// + /// which generates rule patterns that match only the and allowing anything between/before + /// public class MatchProblemValuesPatternFactory: IRulePatternFactory { private MatchWholeStringRulePatternFactory _fallback = new MatchWholeStringRulePatternFactory(); + /// + /// Returns a pattern that matches in . If the word appears at the start/end of the value then ^ or $ is used. When there are multiple failing parts anything is permitted inbweteen i.e. .* + /// + /// + /// + /// public string GetPattern(object sender, Failure failure) { StringBuilder sb = new StringBuilder(); diff --git a/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs index ecf73b8d7..27c108767 100644 --- a/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs @@ -3,8 +3,17 @@ namespace IsIdentifiableReviewer.Out { + /// + /// that generates Regex rule patterns that match the full (entire cell value) only. + /// public class MatchWholeStringRulePatternFactory: IRulePatternFactory { + /// + /// Returns a Regex pattern that matches the full cell value represented by the exactly (with no permitted leading/trailing content) + /// + /// + /// + /// public string GetPattern(object sender,Failure failure) { return "^" + Regex.Escape(failure.ProblemValue) + "$"; diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs index 59f023445..37620b0a9 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs @@ -11,15 +11,35 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Abstract base for classes who act upon by creating new and/or redacting the database. + /// public abstract class OutBase { + /// + /// Existing rules which describe how to detect a that should be handled by this class. These are synced with the contents of the + /// public List Rules { get;} + + /// + /// Persistence of + /// public FileInfo RulesFile { get; } + /// + /// Factory for creating new when encountering novel that do not match any existing rules. May involve user input. + /// public IRulePatternFactory RulesFactory { get; set; } = new MatchWholeStringRulePatternFactory(); + /// + /// Record of changes to (and ). + /// public Stack History = new Stack(); + /// + /// Creates a new instance, populating with the files serialized in + /// + /// Location to load/persist rules from/to. Will be created if it does not exist yet protected OutBase(FileInfo rulesFile) { RulesFile = rulesFile; @@ -86,6 +106,9 @@ protected IsIdentifiableRule Add(Failure f, RuleAction action) return rule; } + /// + /// Removes the last entry from the and . + /// public void Undo() { if(History.Count == 0) diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs index bdad48d04..a2bdb64d1 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs @@ -2,11 +2,26 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Record of a rule added to during the current session and the serialized that was persisted + /// public class OutBaseHistory { + /// + /// The rule generated + /// public IsIdentifiableRule Rule { get; } + + /// + /// The serialized representation of the (added to ) + /// public string Yaml { get; } + /// + /// Records a serialized + /// + /// + /// public OutBaseHistory(IsIdentifiableRule rule, string yaml) { Rule = rule; diff --git a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs index c47b29b79..22b8e6c9e 100644 --- a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs +++ b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs @@ -10,8 +10,19 @@ namespace IsIdentifiableReviewer.Out { + /// + /// + /// Implementation of OutBase for . Base class should + /// be interpreted as rules for detecting which should be redacted. This involves adding a new rule + /// to redact the failure. Then if not in updating the database to perform the redaction. + /// + /// See also: + /// public class RowUpdater : OutBase { + /// + /// Default name for the true positive detection rules (for redacting with). This file will be appended to as new rules are added. + /// public const string DefaultFileName = "RedList.yaml"; /// @@ -27,11 +38,17 @@ public class RowUpdater : OutBase /// The strategy to use to build SQL updates to run on the database /// public IUpdateStrategy UpdateStrategy = new RegexUpdateStrategy(); - + + /// + /// Creates a new instance which stores rules in the (which will also have existing rules loaded from) + /// public RowUpdater(FileInfo rulesFile) : base(rulesFile) { } - + + /// + /// Creates a new instance which stores rules in the + /// public RowUpdater() : this(new FileInfo(DefaultFileName)) { } diff --git a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs index 6bb5805b9..8ca20d339 100644 --- a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs @@ -26,8 +26,14 @@ public enum SymbolsRuleFactoryMode CharactersOnly } + /// + /// Generates Regex patterns for matching based on permutations of digits (\d) and/or characters([A-Z] or [a-z]). See also . + /// public class SymbolsRulesFactory : IRulePatternFactory { + /// + /// Whether to generate Regex match patterns using the permutation of characters, digits or both. + /// public SymbolsRuleFactoryMode Mode { get; set; } /// diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs index 472e19382..9186c8740 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs @@ -5,8 +5,19 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies { + /// + /// Interface for generating SQL statements that perform redactions on a database for a given when applying an + /// public interface IUpdateStrategy { + /// + /// Returns SQL that should be run on to redact the by removing the parts matched in + /// + /// Table on which the SQL should be run + /// Cached primary key knowledge about all tables encountered so far, index with to determine primary keys + /// The cell and value in which a problem value was detected + /// How to redact the + /// IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule); } diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs index bb05414be..a6c4850fc 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs @@ -11,6 +11,14 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies /// public class ProblemValuesUpdateStrategy : UpdateStrategy { + /// + /// Generates 1 UPDATE statement per for redacting the current + /// + /// + /// + /// + /// + /// public override IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule) { diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs index 3a5c82fa5..d5c256244 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs @@ -17,6 +17,14 @@ public class RegexUpdateStrategy : UpdateStrategy { ProblemValuesUpdateStrategy _fallback = new ProblemValuesUpdateStrategy(); + /// + /// Returns SQL for updating the to redact the capture groups in . If no capture groups are represented in the then this class falls back on + /// + /// + /// + /// + /// + /// public override IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule) { if (usingRule == null || string.IsNullOrWhiteSpace(usingRule.IfPattern)) diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs index 878933e68..34fabc673 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs @@ -7,8 +7,19 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies { + /// + /// Abstract implementation of , generates SQL statements for redacting a database + /// public abstract class UpdateStrategy : IUpdateStrategy { + /// + /// Override to generate one or more SQL statements that will fully redact a given in the + /// + /// + /// + /// + /// + /// public abstract IEnumerable GetUpdateSql(DiscoveredTable table,Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule); /// diff --git a/src/applications/IsIdentifiableReviewer/Target.cs b/src/applications/IsIdentifiableReviewer/Target.cs index 2adb72543..6c87a13ca 100644 --- a/src/applications/IsIdentifiableReviewer/Target.cs +++ b/src/applications/IsIdentifiableReviewer/Target.cs @@ -1,19 +1,42 @@ using FAnsi; using FAnsi.Discovery; +using Microservices.IsIdentifiable.Reporting; namespace IsIdentifiableReviewer { + /// + /// The location of a database server for which were detected and redaction may take place + /// public class Target { + /// + /// The user friendly name of this database server + /// public string Name {get;set;} + + /// + /// Connection string for connecting to the server + /// public string ConnectionString { get; set; } + + /// + /// The DBMS type, MySql, Sql Server tec + /// public DatabaseType DatabaseType { get; set; } + /// + /// Returns the + /// + /// public override string ToString() { return Name; } + /// + /// Returns a managed object for the for detecting tables, primary keys, running SQL statements etc + /// + /// public DiscoveredServer Discover() { return new DiscoveredServer(ConnectionString, DatabaseType); diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index b7d2a81d9..efd65fd4d 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -5,6 +5,7 @@ using System.Linq; using IsIdentifiableReviewer.Out; using Microservices.IsIdentifiable.Options; +using Microservices.IsIdentifiable.Reporting; using Microservices.IsIdentifiable.Reporting.Destinations; using Microservices.IsIdentifiable.Reporting.Reports; using Microservices.IsIdentifiable.Rules; @@ -13,6 +14,11 @@ namespace IsIdentifiableReviewer { + /// + /// CLI no user interaction mode for running the reviewer application. In this mode all in a table are + /// run through an existing rules base (for detecting true/false positives) and the database is updated to perform redactions. + /// Any failures not covered by existing rules are routed to + /// public class UnattendedReviewer { private readonly Target _target; @@ -21,15 +27,37 @@ public class UnattendedReviewer private readonly IgnoreRuleGenerator _ignorer; private readonly FileInfo _outputFile; + /// + /// The number of that were redacted in the database. Where there are multiple UPDATE statements run per failure, Updates will only be incremented once. + /// public int Updates = 0; + + /// + /// The number of input that were ignored as false positives based on existing ignore rules + /// public int Ignores = 0; + + /// + /// The number of input that were not covered by any existing rules + /// public int Unresolved = 0; + + /// + /// Total number of processed so far + /// public int Total = 0; private Logger _log; Dictionary _updateRulesUsed = new Dictionary(); Dictionary _ignoreRulesUsed = new Dictionary(); + /// + /// Creates a new instance that will connect to the database server () and perform redactions using the + /// + /// CLI options for the process + /// DBMS to connect to for redacting + /// Rules base for detecting false positives + /// Rules base for redacting true positives public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, IgnoreRuleGenerator ignorer, RowUpdater updater) { _log = LogManager.GetCurrentClassLogger(); @@ -56,6 +84,10 @@ public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, Ign _updater = updater; } + /// + /// Connects to the database and runs all failures through the rules base performing redactions as required + /// + /// public int Run() { //In RulesOnly mode this will be null From b33e91718692169bb7bcfd75412e9c2ea0d2bbe5 Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 3 Sep 2020 11:09:38 +0100 Subject: [PATCH 087/138] Fixed changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78646fb55..bac2c1ed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Changes to MongoDB extraction schema, but backwards compatibility has been tested - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions +- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) ## [1.11.1] - 2020-08-12 @@ -73,7 +74,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Records in the referenced table will blacklist where any UID is found (StudyInstanceUID, SeriesInstanceUID or SOPInstanceUID). This allows blacklisting an entire study or only specific images. - [breaking] Config on live system may need updated - Change the extraction directory generation to be `/image-requests/`. Fixes [MVP Service #159](https://dev.azure.com/smiops/MVP%20Service/_workitems/edit/159/) -- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) ### Fixed From 30ada9215e9f62050473b36af60fdf05a2f6d76c Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 3 Sep 2020 11:11:02 +0100 Subject: [PATCH 088/138] Fixed other bit of changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bac2c1ed3..dacb20a72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions - IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) - +- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` + ## [1.11.1] - 2020-08-12 - Set PublishTrimmed to false to fix bug with missing assemblies in prod. @@ -102,7 +103,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump YamlDotNet from 8.1.0 to 8.1.2 - Bump fo-dicom.Drawing from 4.0.4 to 4.0.5 - Pinned fo-dicom.NetCore to 4.0.5 -- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` ## [1.8.1] - 2020-04-17 From ad648262927bdb15f8e1a426f58260754108177b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2020 05:00:57 +0000 Subject: [PATCH 089/138] Bump mockito-core in /src/common/com.smi.microservices.parent Bumps [mockito-core](https://github.com/mockito/mockito) from 3.5.9 to 3.5.10. - [Release notes](https://github.com/mockito/mockito/releases) - [Commits](https://github.com/mockito/mockito/compare/v3.5.9...v3.5.10) Signed-off-by: dependabot[bot] --- src/common/com.smi.microservices.parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 325eca381..fb79ce2b7 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.5.9 + 3.5.10 From c3d100e8df2089e4aaa737a713ba6ade84a4b9ee Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Fri, 4 Sep 2020 15:54:09 +0100 Subject: [PATCH 090/138] Tidy --- .../Execution/RequestFulfillers/QueryToExecuteResult.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs index 77cf07884..5a2d89fb3 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs @@ -1,4 +1,3 @@ -using JetBrains.Annotations; using System; @@ -22,8 +21,9 @@ public QueryToExecuteResult(string filePathValue, string studyTagValue, string s InstanceTagValue = instanceTagValue; Reject = rejection; RejectReason = rejectionReason; + if (Reject && string.IsNullOrWhiteSpace(RejectReason)) - throw new ArgumentException("RejectReason must be specified if Reject=true"); + throw new ArgumentException("RejectReason must be specified if Reject=true"); } public override string ToString() @@ -45,7 +45,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((QueryToExecuteResult) obj); + return Equals((QueryToExecuteResult)obj); } public override int GetHashCode() From 055dfa389881ec94bfb5ea5560579a528b249ed6 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 10:57:21 +0100 Subject: [PATCH 091/138] Cache values passed to the Validate method to speed up relooks --- .../Runners/IsIdentifiableAbstractRunner.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index b56ee4af2..ff434e9bd 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -88,6 +89,11 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// public List CustomWhiteListRules { get; set; } = new List(); + /// + /// One cache per field in the data being evaluated, records the recent values passed to and the results to avoid repeated lookups + /// + public ConcurrentDictionary> Caches {get;set;} = new ConcurrentDictionary>(); + protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; @@ -220,13 +226,19 @@ public void LoadRules(string yaml) // ReSharper disable once UnusedMemberInSuper.Global public abstract int Run(); + protected IEnumerable Validate(string fieldName, string fieldValue) + { + var cache = Caches.GetOrAdd(fieldName,(v)=>new ConcurrentDictionary()); + return cache.GetOrAdd(fieldValue,(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); + } + /// /// Returns each subsection of which violates validation rules (e.g. the CHI found). /// /// /// /// - protected IEnumerable Validate(string fieldName, string fieldValue) + protected IEnumerable ValidateImpl(string fieldName, string fieldValue) { if (_skipColumns.Contains(fieldName)) yield break; From d715247a77a33aa5276b496651a14300985d1023 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 11:06:14 +0100 Subject: [PATCH 092/138] Added maximum cache size --- .../Runners/IsIdentifiableAbstractRunner.cs | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index ff434e9bd..de81e60f6 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -94,6 +94,21 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// public ConcurrentDictionary> Caches {get;set;} = new ConcurrentDictionary>(); + /// + /// The maximum size of a Cache before we clear it out to prevent running out of RAM + /// + private const int MaxCacheSize = 1_000_000; + + /// + /// The number of values we have attempted to lookup + /// + private int _valuesLookedUp = 0; + + /// + /// Check for exceeding every time this number of lookups has elapsed + /// + private int _checkEvery = 10_000; + protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; @@ -225,15 +240,36 @@ public void LoadRules(string yaml) // ReSharper disable once UnusedMemberInSuper.Global public abstract int Run(); - + + /// + /// Returns each subsection of which violates validation rules (e.g. the CHI found). + /// + /// + /// + /// protected IEnumerable Validate(string fieldName, string fieldValue) { + // every time we lookup a value increment the counter and see if we should check cache overflow + if(_valuesLookedUp++ % _checkEvery == 0) + { + _valuesLookedUp = 0; + foreach (var c in Caches) + { + //any column with more than the maximum number of cached values should be cleared + if(c.Value.Keys.Count > MaxCacheSize) + c.Value.Clear(); + } + } + + // make sure that we have a cache for this column name var cache = Caches.GetOrAdd(fieldName,(v)=>new ConcurrentDictionary()); + + // lookup the cached value or call ValidateImpl return cache.GetOrAdd(fieldValue,(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); } - + /// - /// Returns each subsection of which violates validation rules (e.g. the CHI found). + /// Actual implementation of after a cache miss has occurred. This method is only called when a cached answer is not found for the given and pair /// /// /// From 663cad0ab8637790a1b03e4efdab39209dce3425 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 11:07:38 +0100 Subject: [PATCH 093/138] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df9f219ef..28da3964f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Changes to MongoDB extraction schema, but backwards compatibility has been tested - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions +- Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool ## [1.11.1] - 2020-08-12 From f87b587731dd16fe3ffa2fe523a0742956dd6f77 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 11:13:16 +0100 Subject: [PATCH 094/138] Fix for null values in dictionary cache --- .../Runners/IsIdentifiableAbstractRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index de81e60f6..e8848fe9e 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -265,7 +265,7 @@ protected IEnumerable Validate(string fieldName, string fieldValue) var cache = Caches.GetOrAdd(fieldName,(v)=>new ConcurrentDictionary()); // lookup the cached value or call ValidateImpl - return cache.GetOrAdd(fieldValue,(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); + return cache.GetOrAdd(fieldValue,(k)=>ValidateImpl(fieldName,fieldValue ?? "NULL").ToArray()); } /// From c0573c9de2d254803f38a171f37b843852fa2b2f Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 11:13:57 +0100 Subject: [PATCH 095/138] Fix for null values in dictionary cache (oops) --- .../Runners/IsIdentifiableAbstractRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index e8848fe9e..c6964d59a 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -265,7 +265,7 @@ protected IEnumerable Validate(string fieldName, string fieldValue) var cache = Caches.GetOrAdd(fieldName,(v)=>new ConcurrentDictionary()); // lookup the cached value or call ValidateImpl - return cache.GetOrAdd(fieldValue,(k)=>ValidateImpl(fieldName,fieldValue ?? "NULL").ToArray()); + return cache.GetOrAdd(fieldValue?? "NULL",(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); } /// From f20832445b54ddcd4f8a78b3ed1c269310f188e2 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 11:28:17 +0100 Subject: [PATCH 096/138] Fixed tests to properly pass a field name --- .../IsIdentifiable_TestRegexDetections.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs index 9b5abefbe..f32078b8f 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs @@ -265,6 +265,7 @@ public TestRunner(string valueToTest) : base(new TestOpts()) { _valueToTest = valueToTest; + _fieldToTest = "field"; } public TestRunner(string valueToTest, TestOpts opts, string fieldToTest = "field") From 30fabc3a3a0b57d4ed140f69197627d86bf88561 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 12:05:49 +0100 Subject: [PATCH 097/138] Changed to MemoryCache --- .../Microservices.IsIdentifiable.csproj | 1 + .../Runners/IsIdentifiableAbstractRunner.cs | 36 +++++-------------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index c550d1e1f..1b414f952 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -42,6 +42,7 @@ + diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index c6964d59a..5528875eb 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -11,6 +11,7 @@ using Microservices.IsIdentifiable.Reporting.Reports; using Microservices.IsIdentifiable.Rules; using Microservices.IsIdentifiable.Whitelists; +using Microsoft.Extensions.Caching.Memory; using NLog; using YamlDotNet.Serialization; @@ -92,23 +93,13 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// /// One cache per field in the data being evaluated, records the recent values passed to and the results to avoid repeated lookups /// - public ConcurrentDictionary> Caches {get;set;} = new ConcurrentDictionary>(); + public ConcurrentDictionary Caches {get;set;} = new ConcurrentDictionary(); /// /// The maximum size of a Cache before we clear it out to prevent running out of RAM /// private const int MaxCacheSize = 1_000_000; - - /// - /// The number of values we have attempted to lookup - /// - private int _valuesLookedUp = 0; - - /// - /// Check for exceeding every time this number of lookups has elapsed - /// - private int _checkEvery = 10_000; - + protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; @@ -249,23 +240,14 @@ public void LoadRules(string yaml) /// protected IEnumerable Validate(string fieldName, string fieldValue) { - // every time we lookup a value increment the counter and see if we should check cache overflow - if(_valuesLookedUp++ % _checkEvery == 0) - { - _valuesLookedUp = 0; - foreach (var c in Caches) - { - //any column with more than the maximum number of cached values should be cleared - if(c.Value.Keys.Count > MaxCacheSize) - c.Value.Clear(); - } - } - // make sure that we have a cache for this column name - var cache = Caches.GetOrAdd(fieldName,(v)=>new ConcurrentDictionary()); - + var cache = Caches.GetOrAdd(fieldName,(v)=>new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = MaxCacheSize + })); + // lookup the cached value or call ValidateImpl - return cache.GetOrAdd(fieldValue?? "NULL",(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); + return cache.GetOrCreate(fieldValue?? "NULL",(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); } /// From 3d69e3280ad8f85af27596cbb21f159c58c0747a Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 12:23:47 +0100 Subject: [PATCH 098/138] Added tests and fixed MemoryCache adding --- .../Runners/IsIdentifiableAbstractRunner.cs | 14 +++-- .../IsIdentifiable_TestRegexDetections.cs | 52 ++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index 5528875eb..0af22cf9e 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -238,7 +238,7 @@ public void LoadRules(string yaml) /// /// /// - protected IEnumerable Validate(string fieldName, string fieldValue) + protected virtual IEnumerable Validate(string fieldName, string fieldValue) { // make sure that we have a cache for this column name var cache = Caches.GetOrAdd(fieldName,(v)=>new MemoryCache(new MemoryCacheOptions() @@ -246,8 +246,14 @@ protected IEnumerable Validate(string fieldName, string fieldValue) SizeLimit = MaxCacheSize })); - // lookup the cached value or call ValidateImpl - return cache.GetOrCreate(fieldValue?? "NULL",(k)=>ValidateImpl(fieldName,fieldValue).ToArray()); + //if we have the cached result use it + if(cache.TryGetValue(fieldValue ?? "NULL",out FailurePart[] result)) + return result; + + //otherwise run ValidateImpl and cache the result + return cache.Set(fieldValue?? "NULL", ValidateImpl(fieldName,fieldValue).ToArray(), new MemoryCacheEntryOptions() { + Size=1 + }); } /// @@ -256,7 +262,7 @@ protected IEnumerable Validate(string fieldName, string fieldValue) /// /// /// - protected IEnumerable ValidateImpl(string fieldName, string fieldValue) + protected virtual IEnumerable ValidateImpl(string fieldName, string fieldValue) { if (_skipColumns.Contains(fieldName)) yield break; diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs index f32078b8f..6e98acbab 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs @@ -21,6 +21,36 @@ public void TestChiInString() Assert.AreEqual("0101010101", p.Word); Assert.AreEqual(10, p.Offset); + } + [Test] + public void TestCaching() + { + var runner = new TestRunner("hey there,0101010101 excited to see you"); + runner.Run(); + Assert.AreEqual(1,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(1,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(1,runner.ValidateCalls); + runner.Run(); + + runner.ValueToTest = "ffffff"; + runner.Run(); + Assert.AreEqual(2,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(2,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(2,runner.ValidateCalls); + runner.Run(); + + runner.FieldToTest = "OtherField"; + runner.Run(); + Assert.AreEqual(3,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(3,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(3,runner.ValidateCalls); + runner.Run(); } [TestCase("DD3 7LB")] @@ -256,31 +286,39 @@ public void TestSopDoesNotMatch() private class TestRunner : IsIdentifiableAbstractRunner { - private readonly string _fieldToTest; - private readonly string _valueToTest; + public string FieldToTest {get;set; } + public string ValueToTest {get;set; } public readonly List ResultsOfValidate = new List(); + public int ValidateCalls {get;set;} + public TestRunner(string valueToTest) : base(new TestOpts()) { - _valueToTest = valueToTest; - _fieldToTest = "field"; + ValueToTest = valueToTest; + FieldToTest = "field"; } public TestRunner(string valueToTest, TestOpts opts, string fieldToTest = "field") : base(opts) { - _fieldToTest = fieldToTest; - _valueToTest = valueToTest; + FieldToTest = fieldToTest; + ValueToTest = valueToTest; } public override int Run() { - ResultsOfValidate.AddRange(Validate(_fieldToTest, _valueToTest).OrderBy(v => v.Offset)); + ResultsOfValidate.AddRange(Validate(FieldToTest, ValueToTest).OrderBy(v => v.Offset)); CloseReports(); return 0; } + + protected override IEnumerable ValidateImpl(string fieldName, string fieldValue) + { + ValidateCalls++; + return base.ValidateImpl(fieldName, fieldValue); + } } private class TestOpts : IsIdentifiableAbstractOptions From 3364be57781f1d26be8b1672a6c59abd44c35d42 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 12:31:52 +0100 Subject: [PATCH 099/138] Exposed MaxValidationCacheSize as CLI arg --- .../Options/IsIdentifiableAbstractOptions.cs | 3 +++ .../Runners/IsIdentifiableAbstractRunner.cs | 7 ++++--- .../IsIdentifiable_TestRegexDetections.cs | 13 +++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs b/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs index 4d8168efc..72a93342c 100644 --- a/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs +++ b/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs @@ -63,6 +63,9 @@ public abstract class IsIdentifiableAbstractOptions [Option(HelpText = "Optional. Directory of additional rules in yaml format.")] public string RulesDirectory { get; set; } + [Option(HelpText = "Optional. Maximum number of answers to cache per column.",Default = 1_000_000)] + public int MaxValidationCacheSize {get;set;} = 1_000_000; + /// /// Returns a short string with no spaces or punctuation that describes the target. This will be used /// for naming output reports e.g. "biochemistry" , "mydir" etc diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index 0af22cf9e..516121e94 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -98,13 +98,14 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// /// The maximum size of a Cache before we clear it out to prevent running out of RAM /// - private const int MaxCacheSize = 1_000_000; + public int MaxValidationCacheSize {get;set;} protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; _opts.ValidateOptions(); - + MaxValidationCacheSize = opts.MaxValidationCacheSize; + string targetName = _opts.GetTargetName(); if (opts.ColumnReport) @@ -243,7 +244,7 @@ protected virtual IEnumerable Validate(string fieldName, string fie // make sure that we have a cache for this column name var cache = Caches.GetOrAdd(fieldName,(v)=>new MemoryCache(new MemoryCacheOptions() { - SizeLimit = MaxCacheSize + SizeLimit = MaxValidationCacheSize })); //if we have the cached result use it diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs index 6e98acbab..44eb40509 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs @@ -52,7 +52,20 @@ public void TestCaching() Assert.AreEqual(3,runner.ValidateCalls); runner.Run(); } + [Test] + public void Test_NoCaching() + { + var runner = new TestRunner("hey there,0101010101 excited to see you"); + runner.MaxValidationCacheSize = 0; + runner.Run(); + Assert.AreEqual(1,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(2,runner.ValidateCalls); + runner.Run(); + Assert.AreEqual(3,runner.ValidateCalls); + runner.Run(); + } [TestCase("DD3 7LB")] [TestCase("dd3 7lb")] [TestCase("dd37lb")] From 41d9cae174b7fb28a01ec2a06adfd6b2e301e1a0 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 12:39:36 +0100 Subject: [PATCH 100/138] Added tracking of cache hit/miss rate --- .../Runners/IsIdentifiableAbstractRunner.cs | 15 ++++++ .../IsIdentifiable_TestRegexDetections.cs | 47 ++++++++++--------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index 516121e94..46c3f52d8 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -100,6 +100,16 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// public int MaxValidationCacheSize {get;set;} + /// + /// Total number of calls to that were returned from the cache + /// + public long ValidateCacheHits {get;set;} + + /// + /// Total number of calls to that were missing from the cache and run directly + /// + public long ValidateCacheMisses {get;set;} + protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; @@ -249,7 +259,12 @@ protected virtual IEnumerable Validate(string fieldName, string fie //if we have the cached result use it if(cache.TryGetValue(fieldValue ?? "NULL",out FailurePart[] result)) + { + ValidateCacheHits++; return result; + } + + ValidateCacheMisses++; //otherwise run ValidateImpl and cache the result return cache.Set(fieldValue?? "NULL", ValidateImpl(fieldName,fieldValue).ToArray(), new MemoryCacheEntryOptions() { diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs index 44eb40509..a30079c8b 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs @@ -27,30 +27,36 @@ public void TestCaching() { var runner = new TestRunner("hey there,0101010101 excited to see you"); runner.Run(); - Assert.AreEqual(1,runner.ValidateCalls); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); runner.Run(); - Assert.AreEqual(1,runner.ValidateCalls); - runner.Run(); - Assert.AreEqual(1,runner.ValidateCalls); + Assert.AreEqual(1,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); runner.Run(); + Assert.AreEqual(2,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); runner.ValueToTest = "ffffff"; runner.Run(); - Assert.AreEqual(2,runner.ValidateCalls); - runner.Run(); - Assert.AreEqual(2,runner.ValidateCalls); + Assert.AreEqual(2,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); runner.Run(); - Assert.AreEqual(2,runner.ValidateCalls); + Assert.AreEqual(3,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); runner.Run(); + Assert.AreEqual(4,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); runner.FieldToTest = "OtherField"; runner.Run(); - Assert.AreEqual(3,runner.ValidateCalls); + Assert.AreEqual(4,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); runner.Run(); - Assert.AreEqual(3,runner.ValidateCalls); - runner.Run(); - Assert.AreEqual(3,runner.ValidateCalls); + Assert.AreEqual(5,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); runner.Run(); + Assert.AreEqual(6,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); } [Test] public void Test_NoCaching() @@ -59,11 +65,14 @@ public void Test_NoCaching() runner.MaxValidationCacheSize = 0; runner.Run(); - Assert.AreEqual(1,runner.ValidateCalls); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); runner.Run(); - Assert.AreEqual(2,runner.ValidateCalls); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); runner.Run(); - Assert.AreEqual(3,runner.ValidateCalls); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); runner.Run(); } [TestCase("DD3 7LB")] @@ -304,8 +313,6 @@ private class TestRunner : IsIdentifiableAbstractRunner public readonly List ResultsOfValidate = new List(); - public int ValidateCalls {get;set;} - public TestRunner(string valueToTest) : base(new TestOpts()) { @@ -326,12 +333,6 @@ public override int Run() CloseReports(); return 0; } - - protected override IEnumerable ValidateImpl(string fieldName, string fieldValue) - { - ValidateCalls++; - return base.ValidateImpl(fieldName, fieldValue); - } } private class TestOpts : IsIdentifiableAbstractOptions From 4962ffcba039baa5b53fb1881fdb9a8395960353 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 7 Sep 2020 12:41:31 +0100 Subject: [PATCH 101/138] Added logging of cache hit/miss rate on exit --- .../Runners/IsIdentifiableAbstractRunner.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index 46c3f52d8..4073e5bd8 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -453,6 +453,8 @@ public virtual void Dispose() { foreach (var d in CustomRules.OfType()) d.Dispose(); + + _logger?.Info($"ValidateCacheHits:{ValidateCacheHits} Total ValidateCacheMisses:{ValidateCacheMisses}"); } } } From 636f426b557cbee8abebdf53cf7057abe26dc95d Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 8 Sep 2020 13:09:37 +0100 Subject: [PATCH 102/138] Fixed class diagrams --- .../ProcessDirectory.cd | 87 ++++++++------ src/common/Smi.Common/Messages/Messages.cd | 52 +++------ src/common/Smi.Common/MicroservicesLogging.cd | 55 ++++----- .../Options/Microservices.Common.Options.cd | 28 ----- src/common/Smi.Common/RabbitMQAdapter.cd | 77 +++++++------ src/common/Smi.Common/Smi.Common.csproj | 1 - .../CohortExtractor.cd | 21 ++-- .../DicomRelationalMapper.cd | 12 +- .../IdentifierMapper.cd | 100 +++++++++++------ .../IsIdentifiable.cd | 106 ++++++++++-------- .../MongoDBPopulator.cd | 2 +- .../AllServices.cd | 56 --------- 12 files changed, 273 insertions(+), 324 deletions(-) delete mode 100644 src/common/Smi.Common/Options/Microservices.Common.Options.cd delete mode 100644 tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd diff --git a/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd b/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd index 09b59dbe6..e6e96f01f 100644 --- a/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd +++ b/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd @@ -1,61 +1,78 @@  - - + + + + AAAAAAAAACAAAAAAAAAAAAIAAAAAAQAAIAAAAAAAAAA= + Execution\DicomDirectoryProcessorHost.cs + + + + + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAACACAAAAABAAQAAAAAA= + Options\DicomDirectoryProcessorCliOptions.cs + + + + - - - - Execution\DicomDirectoryFinder.cs - - - - AAAAAQAABAAiAIACAAAAAAQAIAEAAYAAIAAAQCAAAAA= - Execution\DicomDirectoryFinder.cs + AAAAAACAAAICABACAAAgAQgCAAAAAIIAcAAAQCCAAAg= + Execution\DirectoryFinders\DicomDirectoryFinder.cs + - - + + - AAAAAAAAACAAAAAAAAAAAAIAAAAAAQAAIAAAAAAAAAA= - Execution\ProcessDirectoryHost.cs + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBAAAA= + Execution\DirectoryFinders\AccessionDirectoryLister.cs - - - - - - + + - AAAAAAAAAAAAAAAEAAAAAAAAACAAAAAAABAAQAAAAAA= - Options\ProcessDirectoryCliOptions.cs + AAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAACAAAAA= + Execution\DirectoryFinders\BasicDicomDirectoryFinder.cs - - - - - - - - + + + + AAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBAAAA= + Execution\DirectoryFinders\PacsDirectoryFinder.cs + - - + + - + - + + + + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAACAAAAA= + Execution\DirectoryFinders\IDicomDirectoryFinder.cs + + \ No newline at end of file diff --git a/src/common/Smi.Common/Messages/Messages.cd b/src/common/Smi.Common/Messages/Messages.cd index 27ad9a7cf..ab9f3ef9c 100644 --- a/src/common/Smi.Common/Messages/Messages.cd +++ b/src/common/Smi.Common/Messages/Messages.cd @@ -29,34 +29,16 @@ - - - - - - - - - AAQAAAgAAAAAAAAAgAAAABAASAQAAIAAAAAACCAAAAA= + AAAAAAAAAAAAAAAAgAAAAAAAQAAAAIAAAAAAACAAAAA= Messages\Extraction\ExtractFileMessage.cs - - - - - - - - - - - gAAAAAAAAIAAAAAAgAAAAAIAAAAAAIAAAAAAAAAAAAA= + AAAAAAAAAIAAAAAFgAAAAAIAAAAAAIAAAAAAAAAAAAA= Messages\Extraction\ExtractionRequestMessage.cs @@ -64,7 +46,7 @@ - AAAAAAAAAACAAAAAwAAAAAAAAAACAIAAAAAQAAAAAiA= + AAAAAAAAAAAAAAAEwAAAAAAAAAACAIAAAAAQAAAAAiA= Messages\AccessionDirectoryMessage.cs @@ -72,7 +54,7 @@ - AgAAAAAAAACAAAQEwAAAAAAAIBAAAIAAABAQACAAIiA= + AgAAAAAAAACAAAQEwAAAAAAAIBAAAIAIABAQACAAIiA= Messages\DicomFileMessage.cs @@ -88,7 +70,7 @@ - SAAAAAAAEAAAAAAEgAAAAAECAAgDAAAAAEEAAAAIAAA= + CAAAEAAAEAIAAAAMgAAAAAACABgDgIAAAEEAAAAIIiA= Messages\MessageHeader.cs @@ -115,7 +97,7 @@ - AACAAAAAAAACAAAAgAAAAAIAAAAAAIAAAAAAAAAAAAI= + AACAAAAAAAACAAAGgAAAAAAAAAAAAIAAAAAAAAAAAAA= Messages\Extraction\ExtractFileCollectionInfoMessage.cs @@ -123,31 +105,23 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAIAgAEAAAAAAAAAAAAA= + AAAAAAAAAAAAAAAEgAAAQAAABIAAAMACAAAAAAAAAiI= Messages\Extraction\ExtractMessage.cs - - - - - - - - - - + + - AAAAAAAEAAAAAAAAgAAAAQAAAAAAAIAAAAAAAKAAAAI= - Messages\Extraction\ExtractFileStatusMessage.cs + AAAAAAAEAAAAAAAEgAAAAAAAAAAAAIAAAAAAAKCAAiA= + Messages\Extraction\ExtractedFileStatusMessage.cs - AAAAAAAAAAAAAAAAAAAAAAAAAIAgAEAAAAAAAAAAAAA= + AAAAAAAAAAAAAAAAAAAAQAAABIAAAEACAAAAAAAAAAI= Messages\Extraction\IExtractMessage.cs @@ -168,7 +142,7 @@ - SAAAAAAAEAAAAAAAAAAAAAAAAAgCAAAAAEAAAAAIAAA= + CAAAEAAAEAAAAAAAAAAAAAAAAAgCgAAAAEAAAAAIAAA= Messages\IMessageHeader.cs diff --git a/src/common/Smi.Common/MicroservicesLogging.cd b/src/common/Smi.Common/MicroservicesLogging.cd index 1ab42de5d..100c8b65f 100644 --- a/src/common/Smi.Common/MicroservicesLogging.cd +++ b/src/common/Smi.Common/MicroservicesLogging.cd @@ -1,7 +1,7 @@  - + @@ -13,7 +13,7 @@ - + @@ -21,67 +21,60 @@ - AAAAAAAAACIAAAAAAAAAAAAAAAAAAAAAIAAAAAEEgAA= + AAQAAAACBDIAEAAAAAAIAAEAAgAAAAhAIAAACAQEgAI= Execution\MicroserviceHost.cs - - - - - - - - AAAAEAAAAALABAAAAAAAAAAAAEAECAAAAAAAAAAEAEA= - Messaging\Consumer.cs - - - - AAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAABAAAAAA= + AAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAA= Execution\MicroserviceHostBootstrapper.cs - - - - - + + + + AAAAAAAAiALABAAAGAAAAQAAAEAEgABIAAgAAAAEAEA= + Messaging\Consumer.cs + + + + + - BAAAAAAAAAAAAAAAQiAAAAAAIAAAAAAAAAAAAAAAIAA= - ProducerModel.cs + BAAAAEAAAAAEAAAAQiAAABCAIAAAAABAAAAAAAAEAQA= + Messaging\ProducerModel.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAA= + AAAAAAAACAAAAAAAAAAAAQAAAAAEAABAAAAAAAAAAAA= Messaging\IConsumer.cs - SAAAAAAAEAAAAAAAAAAAAAAAAAgCAAAAAEAAAAAIAAA= + CAAAEAAAEAAAAAAAAAAAAAAAAAgCgAAAAEAAAAAIAAA= Messages\IMessageHeader.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= Messages\IMessage.cs - - + + - AAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAA= - IProducerModel.cs + AAAAAEAAAAAAAAAAACAAAAAAAAAAAABAAAAAAAAAAAA= + Messaging\IProducerModel.cs diff --git a/src/common/Smi.Common/Options/Microservices.Common.Options.cd b/src/common/Smi.Common/Options/Microservices.Common.Options.cd deleted file mode 100644 index ae6c320f8..000000000 --- a/src/common/Smi.Common/Options/Microservices.Common.Options.cd +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - AAAAAAAAAAAAEgAAAAAAQQAAAAAQAAAAAAACAIAAAAA= - Options\MicroservicesOptions.cs - - - - - - - - - AAAAAEIAEAAAEAAAgAAAAAAAAAAAAIAAAAgEAIAAAAA= - Options\ConsumerOptions.cs - - - - - - AAAAAAAAEAAAEAAAgAgAAAAAAAAAAIAAAAAAAIAAAAA= - Options\ProducerOptions.cs - - - - \ No newline at end of file diff --git a/src/common/Smi.Common/RabbitMQAdapter.cd b/src/common/Smi.Common/RabbitMQAdapter.cd index a2056baf7..3323774e6 100644 --- a/src/common/Smi.Common/RabbitMQAdapter.cd +++ b/src/common/Smi.Common/RabbitMQAdapter.cd @@ -1,59 +1,72 @@  - - - - AAgAAAAIAFCAAAQAIAAAAQAAAAAAAAAABAACAAQEBAA= - RabbitMQAdapter.cs - - - - - - - - - - - - AAAAAAAAAAAAEgAAAAAAQQAAAAAQAAAAAAACAIAAAAA= - Options\MicroservicesOptions.cs - - - - - - AAAAAAAAEAAAEAAAgAgAAAAAAAAAAIAAAAAAAIAAIAA= + AAAAAAAAAAAAAAAEgAAEAAAAAAAAAIAAAAAAAAAAImA= Options\ProducerOptions.cs - AAAAAEIAEAAAEAAAgAAAAAAAAAAAAIAAAAgEAIAAIAA= + AAAAAEAAAAAAAAAEAAAAAAAAAAAAAQAAAAgAAAAAIAA= Options\ConsumerOptions.cs - - + + + + + + RabbitMQAdapter.cs + + + + + + RabbitMQAdapter.cs + + + + + RabbitMQAdapter.cs + + + + + AAAAAQAIRJCAAAAEoACwIQAAIAAAAAAIBEACAgAEAAA= + RabbitMQAdapter.cs + + + + + + + + - AAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAA= - IProducerModel.cs + AAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAgAAAAAA= + Options\CliOptions.cs - + - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAA= + AAAAAAAACAAAAAAAAAAAAQAAAAAEAABAAAAAAAAAAAA= Messaging\IConsumer.cs + + + + AAAAAEAAAAAAAAAAACAAAAAAAAAAAABAAAAAAAAAAAA= + Messaging\IProducerModel.cs + + \ No newline at end of file diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 93de313f5..9ddf36651 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -21,7 +21,6 @@ - diff --git a/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd b/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd index 1b48c05d8..21957cc37 100644 --- a/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd +++ b/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd @@ -16,7 +16,7 @@ - + @@ -27,7 +27,7 @@ - AAAAAAAEAAICAAAAAAAAAAAAABAAAAAAAEQAAAAAAAA= + AAAAAAAEAAICAAAAAAAAAAAAABAAAgAAAEQAAAAAAAA= Execution\RequestFulfillers\FromCataloguesExtractionRequestFulfiller.cs @@ -45,7 +45,7 @@ - AAAAAAAICCAAAgAAAAAAAAAAAAAAAAAAABACAAAAAAA= + AAAAAAAICCAAAgAAAAAAAEAAAAAAAAAAIDACAAAAAAA= Execution\CohortExtractorHost.cs @@ -55,7 +55,7 @@ - AAAAAAAIAAKAAAEAAgAAAEAAAAAAAAAAABAAAAAAAAA= + AAAAAAAIAQKAAAEAAAAAAEAAAAAAAAAAABAAAAAAAAA= Messaging\ExtractionRequestQueueConsumer.cs @@ -65,7 +65,7 @@ - + AgAAQAAAAAQCAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAA= Execution\ExtractImageCollection.cs @@ -74,7 +74,7 @@ - AAAAAAAEAEAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA= + AAAAAAAEAAAAAAAAAAAAAAAAAAAQAAAAAAAAAQAAAAA= Execution\ProjectPathResolvers\DefaultProjectPathResolver.cs @@ -136,19 +136,16 @@ - + - AAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA= + AAAAAAAAAAACAAAAAAAAAAAAAAAAAgAAAEAAAAAAAAA= Execution\RequestFulfillers\IExtractionRequestFulfiller.cs - - - - AAAAAAAEAEAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA= + AAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= Execution\ProjectPathResolvers\IProjectPathResolver.cs diff --git a/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd b/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd index 0f7ce2bbe..65364fd8a 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd +++ b/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd @@ -4,10 +4,10 @@ - + - + @@ -41,7 +41,7 @@ - AAEAAAAAAAAAAAAAAAAAAAAAIAQEAAAAEAAAAAAAAAA= + AAEAAAACAAAAAAAAAAAAAAAAIAQEAAAAEAAAAAAAAAA= Execution\DicomFileMessageToDatasetListProvider.cs @@ -58,7 +58,7 @@ - + AAAAAAAAAAAAAABAAAAAAAACIEAAAAAAAAAAgAAAAAA= Execution\NLogThrowerDataLoadEventListener.cs @@ -123,8 +123,8 @@ Execution\Namers\ICanCreateStagingMyself.cs - - + + diff --git a/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd b/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd index 2054affeb..064a6c26a 100644 --- a/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd +++ b/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd @@ -3,7 +3,7 @@ - AAAAAAAAACAAAgAAAAAAAAAAAAAAAAAAAAACAAAAAAA= + ABACAAAAACAAAgAAAAAAQAAAAAAAAAAAIAACAAAAAAA= Execution\IdentifierMapperHost.cs @@ -16,29 +16,74 @@ AAAAAAAAAAAAAAAAAAAAQAAAIAAAAAIAAAAAAAAAAAA= Messaging\IdentifierMapperControlMessageHandler.cs + + + + + + AAAAAAAAAADAAAAAIAAAQQAQAEAAAAAAAAAAAwAAAAA= + Messaging\IdentifierMapperQueueConsumer.cs + + + + + + AAAAAAAACSAAAgAAAAAAAAgAIAAAQAQAAAAACAAgAAA= + Execution\Swappers\ForGuidIdentifierSwapper.cs + - - + + - AAAAAAAACSAAAgAAAAAAAAAAIAAAQAAAAAAACAAgAAA= - Execution\ForGuidIdentifierSwapper.cs + AAAAAAAACSAAAAAAAAAAAAAAICAAAAAAAAAAiAAAAAA= + Execution\Swappers\PreloadTableSwapper.cs + + + + + + AAAAEAAACqAAAAAgAAAAAAAAIAAAAAAAAABACAAAEAA= + Execution\Swappers\RedisSwapper.cs - - + + - AAAAAAAACSAAAAAAAAAAAAAAACAAAAAAAAAAiAAAAAA= - Execution\IdentifierSwapper.cs + AAAAAAAAGSAAAAAAAAAAAAgAAgACAAAAABBACAAAAAA= + Execution\Swappers\SwapIdentifiers.cs - - - + + + + AAAAAAEgCSAAAAAAQAAAAAAAIAAAAAAAAAAACAAAACA= + Execution\Swappers\TableLookupSwapper.cs + + + + + + AAAAAAAECGAQAAAAAAAAAAAAIAAACAAAAABACAAAAAA= + Execution\Swappers\TableLookupWithGuidFallbackSwapper.cs + + + + + + + + + + + + + + - - + + @@ -52,30 +97,11 @@ - - - - - - - - - - - - - EAAAAAAAAADAAgAAAABAQAAAAAAEAAAAAAAAAAAAAAA= - Messaging\IdentifierMapperQueueConsumer.cs - - - - - - - + + - AAAAAAAACCAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAA= - Execution\ISwapIdentifiers.cs + AAAAAAAACCAAAAAAAAAAAAAAAAAAAAAAAABACAAAAAA= + Execution\Swappers\ISwapIdentifiers.cs diff --git a/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd b/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd index fdd9fbbdf..98a0e3fd2 100644 --- a/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd +++ b/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd @@ -12,17 +12,10 @@ - - - - AAAAAgAAAAAABAAAAAAAEAAAAAAASAAAAAAAQAAAQAA= - NerEngine.cs - - - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAQAAAAAAgAAAAAABAAAAIAAACAAAAAACAAAAIAI= Runners\DatabaseRunner.cs @@ -30,25 +23,29 @@ - - + + + + + + + - oAAAAAACAACAAwAABCBAAIAAIACoAAAAAAAAAAAAYBI= + IAAAAAAqACDAA0AABBBEwIAAIACoAABAAABAAADEYAI= Runners\IsIdentifiableAbstractRunner.cs - - - + + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAIAAACAAEIEAAAAAAAAAIAAAAABAAABgAAAAAAA= Reporting\Destinations\CsvDestination.cs @@ -74,14 +71,14 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAIAAAEAAAAAgAAAAAAAAAAAAAAAAAAAAABAA= Reporting\Destinations\DatabaseDestination.cs - AFAABAAAgABAABAAAQAACAAAAAAGAIkABACAAAEEACI= + AFAABAAAgAAAABAAAQAACAAAAAQGBIkABACAAAEEACI= Options\IsIdentifiableAbstractOptions.cs @@ -114,29 +111,19 @@ - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AIAAgAgAAASgAAAAggAAAAACIAAIAAABAAAAAAIAKAI= Runners\DicomFileRunner.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAIAAAAAAAAAUAAAAAAEQAAIAEAAQIAIAACIAEAoAI= Runners\IsIdentifiableMongoRunner.cs - - - - AAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAgAA= - Failure\FailurePart.cs - - - - - @@ -151,56 +138,76 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAAEAAAAAAAAAAAEAAAAAAAABAIBQAAAQACAAAAAA= Reporting\Reports\FailureStoreReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAAAAAAAAAAAAAAABAAAAAAAAAAAAAgAwAAAAEAAA= Reporting\Reports\ColumnFailureReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAA= Reporting\Reports\FailingValuesReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQACAAAAAA= Reporting\Reports\PixelTextFailureReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + BAIBAAAAAAAAAAAAAAAAAAAAAAEIIAAAABQQCAEAAEA= Reporting\Reports\TreeFailureReport.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAEAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAA= Whitelists\CsvWhitelist.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAAAAAAIAAAAAAIAAAAAAAAABAAAAAAAAAAAA= Whitelists\DiscoveredColumnWhitelist.cs + + + + AAEAAAAAAAAAAAAAwAQAAAAAAAAAAIAAAAAAAAAAgAA= + Failures\FailurePart.cs + + + + + + + + + + + BAAAgAgAABAAIACAAAAAAAACgAAAQAAAAAAAAAAAAIQ= + Rules\IsIdentifiableRule.cs + + + - + AAIAAAAAAAAAAAAAAABAAAAAAAAAIAAAAAAAAAAAQAA= Reporting\Reports\IFailureReport.cs @@ -209,22 +216,29 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAgAAAAAAA= Reporting\Destinations\IReportDestination.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAA= Whitelists\IWhitelistSource.cs - - + + + + AAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Rules\ICustomRule.cs + + + + AIAQAAAAAAAAAAAAAAIAgAAAAAAAAQgBAAAACAEAIQA= - Failure\FailureClassification.cs + Failures\FailureClassification.cs diff --git a/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd b/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd index 56f8806f9..5d3f1b38c 100644 --- a/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd +++ b/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd @@ -1,7 +1,7 @@  - + AAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAIAACAAAAAAA= Execution\MongoDbPopulatorHost.cs diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd b/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd deleted file mode 100644 index f4e0ad88b..000000000 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 1078534ab20d1f55e267d0dd17ca9681fa5a4050 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2020 05:39:12 +0000 Subject: [PATCH 103/138] Bump Microsoft.Extensions.Caching.Memory from 3.1.7 to 3.1.8 Bumps [Microsoft.Extensions.Caching.Memory](https://github.com/aspnet/Extensions) from 3.1.7 to 3.1.8. - [Release notes](https://github.com/aspnet/Extensions/releases) - [Commits](https://github.com/aspnet/Extensions/compare/v3.1.7...v3.1.8) Signed-off-by: dependabot[bot] --- .../Microservices.IdentifierMapper.csproj | 2 +- .../Microservices.IsIdentifiable.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj index 5b4597df3..346ef70a6 100644 --- a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj +++ b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index 1b414f952..9e5436458 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -42,7 +42,7 @@ - + From ab060d247033abed0464b4cd25058b1fea1d8be6 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 08:55:17 +0100 Subject: [PATCH 104/138] Removed version and empty end column from Packages.md For #64 --- PACKAGES.md | 60 +++++++++---------- .../Smi.Common.Tests/NuspecIsCorrectTests.cs | 4 -- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/PACKAGES.md b/PACKAGES.md index aa85131b7..f80452f56 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -7,33 +7,33 @@ 2. This package is widely used and is actively maintained. 3. It is open source. -| Package | Source Code | Version | License | Purpose | Additional Risk Assessment | -| ------- | ------------| --------| ------- | ------- | -------------------------- | -| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.8.0](https://www.nuget.org/packages/CommandLineParser/2.8.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | -| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.5](https://www.nuget.org/packages/CsvHelper/15.0.5) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | -| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | -| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.1](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | -| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.5](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.10](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.10) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | -| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.8](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.8) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | -| JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | -| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | -| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | -| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.7](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL | -| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.1](https://www.nuget.org/packages/MongoDB.Driver/2.11.1)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.1/License) | For writting/reading dicom tags into MongoDb databases| -| NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | -| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | -| SecurityCodeScan | [GitHub](https://security-code-scan.github.io/) | [3.5.3](https://www.nuget.org/packages/SecurityCodeScan/3.5.3) | [LGPL 3.0](https://opensource.org/licenses/lgpl-3.0.html) | Scans code for security issues during build | | -| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.1.58](https://www.nuget.org/packages/StackExchange.Redis/2.1.58) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | | -| Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | | -| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Drawing.Common/4.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | | -| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [12.1.1](https://www.nuget.org/packages/System.IO.Abstractions/12.1.1) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | | -| System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | File I/O | | -| System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | -| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.89.4](https://www.nuget.org/packages/Terminal.Gui/0.89.4) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | | -| Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | -| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [8.1.2](https://www.nuget.org/packages/YamlDotNet/8.1.2) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.6](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.6) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | -| coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [1.4.2](https://www.nuget.org/packages/coveralls.io/1.4.2) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io | | -| OpenCover | [GitHub](https://github.com/OpenCover/opencover) | [4.7.922](https://www.nuget.org/packages/OpenCover/4.7.922) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE) | Calculates code coverage for tests| | +| Package | Source Code | License | Purpose +| ------- | ------------| ------- | ------- | +| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | +| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | +| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [MS-PL](https://opensource.org/licenses/MS-PL) | | +| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | +| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | +| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | +| JetBrains.Annotations | |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | +| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | +| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | +| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL +| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) | [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.1/License) | For writting/reading dicom tags into MongoDb databases +| NLog | [GitHub](https://github.com/NLog/NLog) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | +| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission +| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | +| SecurityCodeScan | [GitHub](https://security-code-scan.github.io/) | [LGPL 3.0](https://opensource.org/licenses/lgpl-3.0.html) | Scans code for security issues during build | +| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | +| Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | +| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | +| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | +| System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) |[MIT](https://opensource.org/licenses/MIT) | File I/O | +| System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| +| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | +| Tesseract | [GitHub](https://github.com/charlesw/tesseract/) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| +| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files +| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | +| coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io | +| OpenCover | [GitHub](https://github.com/OpenCover/opencover) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE) | Calculates code coverage for tests| diff --git a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs index 88f9c6e79..37d1e65ba 100644 --- a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs +++ b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs @@ -118,10 +118,6 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) { int count = new Regex(Regex.Escape(version)).Matches(line).Count; - - Assert.GreaterOrEqual(count, 2, - "Markdown file {0} did not contain 2 instances of the version {1} for package {2} in {3}", - packagesMarkdown, version, package, csproj); found = true; } } From dd55ba1333def8eaaeee866391a228c15f2d7228 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Wed, 9 Sep 2020 08:59:16 +0100 Subject: [PATCH 105/138] Bump Microsoft.Extensions.Caching.Memory from 3.1.7 to 3.1.8 --- PACKAGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PACKAGES.md b/PACKAGES.md index aa85131b7..20beb351f 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -19,7 +19,7 @@ | JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | | Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | | Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.7.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | -| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.7](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL | +| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.8](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.8) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.8/License) | Caching ID mappings retrieved from Redis/MySQL | | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.1](https://www.nuget.org/packages/MongoDB.Driver/2.11.1)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.1/License) | For writting/reading dicom tags into MongoDb databases| | NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | | Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | From da8b7df74b0036062f19952465aa5513c281c216 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 09:09:53 +0100 Subject: [PATCH 106/138] Moved logs root to GlobalOptions --- CHANGELOG.md | 5 +++++ README.md | 14 -------------- src/common/Smi.Common/Options/GlobalOptions.cs | 3 ++- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28da3964f..5d870a070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions - Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool + +### Changed +- Environment variables are no longer used. Previous settings now appear in configuration file + - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` + ## [1.11.1] - 2020-08-12 - Set PublishTrimmed to false to fix bug with missing assemblies in prod. diff --git a/README.md b/README.md index 82cdaad2b..ca33497bd 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ The latest binaries can be downloaded from the [releases section](https://github 1. [Microservices](#microservices) 1. [Data Load Microservices](#data-load-microservices) 2. [Image Extraction Microservices](#image-extraction-microservices) -1. [Environment Variables](#environment-variables) 2. [Solution Overivew](#solution-overview) 3. [Building](#building) 4. [Testing](#testing) @@ -74,19 +73,6 @@ A control queue is provided for controlling Microservices during runtime. It su | Fatal Error Logging | All Microservices that crash or log a fatal error are shut down and log a message to the Fatal Error Logging Exchange. TODO: Nobody listens to this currently.| | Quarantine | TODO: Doesn't exist yet.| -## Environment Variables - -The following environment variables are expected by the program: - ->TODO can we move `ISIDENTIFIABLE_NUMTHREADS` to config yaml/CLI? - -| Name | Purpose | Example | -|------|---------|---------| -| SMI_LOGS_ROOT | Required to be set if `ForceSmiLogsRoot` is set to `true` in the service config. Determines where log files are written to | `/var/log/smi` | -| MONGO_SERVICE_PASSWORD | Password that should be used to access the MongoDb database used by ETL pipeline services | MyPassword| -| ISIDENTIFIABLE_NUMTHREADS | Optional (defaults to 1). The number of threads to use when looking for identifiable data in extraction pipeline | 1| -| CI | When running tests in a CI, this ensures that integration tests are failed instead of skipped | 1| - ## Solution Overview Appart from the Microservices (documented above) the following library classes are also included in the solution: diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 2fcc49a31..f6680b03b 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -90,6 +90,7 @@ private GlobalOptions() { } public DeadLetterReprocessorOptions DeadLetterReprocessorOptions { get; set; } public IsIdentifiableOptions IsIdentifiableOptions { get; set; } + public string LogsRoot { get; set; } #endregion @@ -534,7 +535,7 @@ public override string ToString() public class FileSystemOptions { /// - /// If set, services will require that the "SMI_LOGS_ROOT" environment variable is set and points to a valid directory. + /// If set, services will require that is set and points to a valid directory. /// This helps to ensure that we log to a central location on the production system. /// public bool ForceSmiLogsRoot { get; set; } = false; From 96942b99c4be354ff72462216f7c43aec110af4e Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 09:14:25 +0100 Subject: [PATCH 107/138] Changed usages to use globals.LogsRoot --- src/common/Smi.Common/Execution/MicroserviceHost.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/Smi.Common/Execution/MicroserviceHost.cs b/src/common/Smi.Common/Execution/MicroserviceHost.cs index 7f0c7ef5b..784a70543 100644 --- a/src/common/Smi.Common/Execution/MicroserviceHost.cs +++ b/src/common/Smi.Common/Execution/MicroserviceHost.cs @@ -74,10 +74,10 @@ protected MicroserviceHost( if (globals.FileSystemOptions.ForceSmiLogsRoot) { - string smiLogsRoot = Environment.GetEnvironmentVariable("SMI_LOGS_ROOT"); + string smiLogsRoot = globals.LogsRoot; if (string.IsNullOrWhiteSpace(smiLogsRoot) || !Directory.Exists(smiLogsRoot)) - throw new ApplicationException($"Invalid logs root: SMI_LOGS_ROOT={smiLogsRoot}"); + throw new ApplicationException($"Invalid logs root: {smiLogsRoot}"); LogManager.Configuration.Variables["baseFileName"] = $"{smiLogsRoot}/{HostProcessName}/${{cached:cached=true:clearCache=None:inner=${{date:format=yyyy-MM-dd-HH-mm-ss}}}}-${{processid}}"; From 2772be7b74944654cf1849c5f8479d1f8fbe3546 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 09:17:19 +0100 Subject: [PATCH 108/138] Removed pointless count variable --- tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs index 37d1e65ba..1d03ef28f 100644 --- a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs +++ b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs @@ -117,7 +117,6 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM { if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) { - int count = new Regex(Regex.Escape(version)).Matches(line).Count; found = true; } } From 4a5eeed3d8a7b889a3861831fa7c2f0b01da1773 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Wed, 9 Sep 2020 08:19:34 +0000 Subject: [PATCH 109/138] Small tidyup, remove disused count of regex matches in line --- .../Smi.Common.Tests/NuspecIsCorrectTests.cs | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs index 37d1e65ba..e3ae591d8 100644 --- a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs +++ b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs @@ -92,13 +92,11 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM string packageDependency = d.Groups[1].Value; string versionDependency = d.Groups[2].Value; - if (packageDependency.Equals(package)) - { - Assert.AreEqual(version, versionDependency, - "Package {0} is version {1} in {2} but version {3} in {4}", package, version, csproj, - versionDependency, nuspec); - found = true; - } + if (!packageDependency.Equals(package)) continue; + Assert.AreEqual(version, versionDependency, + "Package {0} is version {1} in {2} but version {3} in {4}", package, version, csproj, + versionDependency, nuspec); + found = true; } if (!found) @@ -110,23 +108,20 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM //And make sure it appears in the packages.md file - if (packagesMarkdown != null) + if (packagesMarkdown == null) continue; + found = false; + foreach (string line in File.ReadAllLines(packagesMarkdown)) { - found = false; - foreach (string line in File.ReadAllLines(packagesMarkdown)) + if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) { - if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) - { - int count = new Regex(Regex.Escape(version)).Matches(line).Count; - found = true; - } + found = true; } - - if (!found) - Assert.Fail("Package {0} in {1} is not documented in {2}. Recommended line is:\r\n{3}", package, - csproj, packagesMarkdown, - BuildRecommendedMarkdownLine(package, version)); } + + if (!found) + Assert.Fail("Package {0} in {1} is not documented in {2}. Recommended line is:\r\n{3}", package, + csproj, packagesMarkdown, + BuildRecommendedMarkdownLine(package, version)); } } From 3e876084b1cd116e7dae7841c9bba6a6f2ce9328 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 09:28:02 +0100 Subject: [PATCH 110/138] Moved mongodb password to config --- CHANGELOG.md | 1 + src/common/Smi.Common.MongoDb/MongoClientHelpers.cs | 9 +++------ src/common/Smi.Common/Options/GlobalOptions.cs | 3 +++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d870a070..17bb58a3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Environment variables are no longer used. Previous settings now appear in configuration file - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` + - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` ## [1.11.1] - 2020-08-12 diff --git a/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs b/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs index bd97f0cf2..b72ad8b01 100644 --- a/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs +++ b/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs @@ -10,7 +10,6 @@ namespace Smi.Common.MongoDB { public static class MongoClientHelpers { - private const string MongoServicePasswordVar = "MONGO_SERVICE_PASSWORD"; private const string AuthDatabase = "admin"; // Always authenticate against the admin database private static readonly ILogger _logger = LogManager.GetCurrentClassLogger(); @@ -37,12 +36,10 @@ public static MongoClient GetMongoClient(MongoDbOptions options, string applicat WriteConcern = new WriteConcern(journal: true) }); - string password = Environment.GetEnvironmentVariable(MongoServicePasswordVar, EnvironmentVariableTarget.Process); + if (string.IsNullOrWhiteSpace(options.Password)) + throw new ApplicationException($"MongoDB password must be set"); - if (string.IsNullOrWhiteSpace(password)) - throw new ApplicationException($"MongoDB password must be set in \"{MongoServicePasswordVar}\""); - - MongoCredential credentials = MongoCredential.CreateCredential(AuthDatabase, options.UserName, password); + MongoCredential credentials = MongoCredential.CreateCredential(AuthDatabase, options.UserName, options.Password); var mongoClientSettings = new MongoClientSettings { diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index f6680b03b..0a28079bb 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -487,6 +487,9 @@ public class MongoDbOptions /// UserName for authentication. If empty, authentication will be skipped. /// public string UserName { get; set; } + + public string Password {get;set;} + public string DatabaseName { get; set; } public bool AreValid(bool skipAuthentication) From 82ec6aac7b2fec40e9447bcb8be190ff2bfc3314 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 09:28:20 +0100 Subject: [PATCH 111/138] Removed ISIDENTIFIABLE_NUMTHREADS --- CHANGELOG.md | 1 + .../Microservices.IsIdentifiable/Runners/DatabaseRunner.cs | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17bb58a3e..10d4d30ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Environment variables are no longer used. Previous settings now appear in configuration file - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` + - Removed `ISIDENTIFIABLE_NUMTHREADS` as it didn't work correctly anyway ## [1.11.1] - 2020-08-12 diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs index faa714e99..04b462ed7 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs @@ -49,10 +49,7 @@ public override int Run() var reader = cmd.ExecuteReader(); - // The query can run in parallel, configure using ISIDENTIFIABLE_NUMTHREADS env var - // XXX default is single-threaded because it breaks during NERd otherwise. - int numThreads = int.Parse(Environment.GetEnvironmentVariable("ISIDENTIFIABLE_NUMTHREADS") ?? "1"); - foreach (Reporting.Failure failure in reader.Cast().AsParallel().WithDegreeOfParallelism(numThreads).SelectMany(GetFailuresIfAny)) + foreach (Failure failure in reader.Cast().SelectMany(GetFailuresIfAny)) AddToReports(failure); CloseReports(); From 85b9d5d72b7164180e9cccf41aa444335e672b7f Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 10:06:08 +0100 Subject: [PATCH 112/138] Switched from static to factory/decorator pattern --- .../Program.cs | 2 +- .../Smi.Common/Options/GlobalOptions.cs | 45 +----- .../Options/Microservices.Common.Options.cd | 152 +++++++++++++++++- .../Smi.Common/Options/OptionsDecorator.cs | 14 ++ .../Smi.Common/Options/OptionsFactory.cs | 73 +++++++++ .../Options/RabbitMqConfigOptions.md | 4 +- src/common/Smi.Common/README.md | 6 +- .../Microservices.CohortExtractor/Program.cs | 2 +- .../Microservices.CohortPackager/Program.cs | 2 +- .../Program.cs | 2 +- .../Program.cs | 2 +- .../Microservices.DicomReprocessor/Program.cs | 2 +- .../Microservices.DicomTagReader/Program.cs | 2 +- .../Microservices.FileCopier/Program.cs | 2 +- .../Microservices.IdentifierMapper/Program.cs | 2 +- .../Microservices.IsIdentifiable/Program.cs | 4 +- .../Microservices.MongoDbPopulator/Program.cs | 2 +- .../MongoQueryParserTests.cs | 2 +- .../DeadLetterTestHelper.cs | 2 +- .../HeaderPreservationTest.cs | 2 +- tests/common/Smi.Common.Tests/OptionsTests.cs | 6 +- .../Smi.Common.Tests/RabbitMqAdapterTests.cs | 4 +- .../ExtractionRequestQueueConsumerTest.cs | 4 +- .../Execution/CohortPackagerHostTest.cs | 4 +- .../DLEBenchmarkingTests/HowFastIsDLETest.cs | 4 +- .../DicomRelationalMapperHostTests.cs | 2 +- .../DicomRelationalMapperTests.cs | 2 +- .../MicroservicesIntegrationTest.cs | 2 +- .../RunMeFirstTests/RunMeFirstMongoServers.cs | 2 +- .../DicomTagReaderTestHelper.cs | 2 +- .../Execution/FileCopierHostTest.cs | 2 +- .../IdentifierMapperTests.cs | 4 +- .../ServiceTests/IsIdentifiableHostTests.cs | 8 +- .../ImageMessageProcessorTests_NoMongo.cs | 2 +- .../MongoDbPopulatorTestHelper.cs | 2 +- 35 files changed, 283 insertions(+), 91 deletions(-) create mode 100644 src/common/Smi.Common/Options/OptionsDecorator.cs create mode 100644 src/common/Smi.Common/Options/OptionsFactory.cs diff --git a/src/applications/Applications.DicomDirectoryProcessor/Program.cs b/src/applications/Applications.DicomDirectoryProcessor/Program.cs index 05bc8ccd4..4d04a590f 100644 --- a/src/applications/Applications.DicomDirectoryProcessor/Program.cs +++ b/src/applications/Applications.DicomDirectoryProcessor/Program.cs @@ -26,7 +26,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( processDirectoryOptions => { - GlobalOptions globalOptions = GlobalOptions.Load(processDirectoryOptions); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load(processDirectoryOptions); var bootStrapper = new MicroserviceHostBootstrapper(() => new DicomDirectoryProcessorHost(globalOptions, processDirectoryOptions)); return bootStrapper.Main(); diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 0a28079bb..8fba1d990 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -20,50 +20,7 @@ namespace Smi.Common.Options { public class GlobalOptions { - public static GlobalOptions Load(string environment = "default", string currentDirectory = null) - { - IDeserializer deserializer = new DeserializerBuilder() - .WithObjectFactory(GetGlobalOption) - .IgnoreUnmatchedProperties() - .Build(); - - currentDirectory = currentDirectory ?? Environment.CurrentDirectory; - - // Make sure environment ends with yaml - if (!(environment.EndsWith(".yaml") || environment.EndsWith(".yml"))) - environment += ".yaml"; - - // If the yaml file doesn't exist and the path is relative, try looking in currentDirectory instead - if (!File.Exists(environment) && !Path.IsPathRooted(environment)) - environment = Path.Combine(currentDirectory, environment); - - string text = File.ReadAllText(environment); - - var globals = deserializer.Deserialize(new StringReader(text)); - globals.CurrentDirectory = currentDirectory; - globals.MicroserviceOptions = new MicroserviceOptions(); - - return globals; - } - - public static GlobalOptions Load(CliOptions cliOptions) - { - GlobalOptions globalOptions = Load(cliOptions.YamlFile); - globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); - - return globalOptions; - } - - - private static object GetGlobalOption(Type arg) - { - return arg == typeof(GlobalOptions) ? - new GlobalOptions() : - Activator.CreateInstance(arg); - } - - private GlobalOptions() { } - + #region AllOptions /// diff --git a/src/common/Smi.Common/Options/Microservices.Common.Options.cd b/src/common/Smi.Common/Options/Microservices.Common.Options.cd index ae6c320f8..d6f8c7dcf 100644 --- a/src/common/Smi.Common/Options/Microservices.Common.Options.cd +++ b/src/common/Smi.Common/Options/Microservices.Common.Options.cd @@ -13,16 +13,164 @@ - AAAAAEIAEAAAEAAAgAAAAAAAAAAAAIAAAAgEAIAAAAA= + AAAAAEAAAAAAAAAEAAAAAAAAAAAAAQAAAAgAAAAAIAA= Options\ConsumerOptions.cs - AAAAAAAAEAAAEAAAgAgAAAAAAAAAAIAAAAAAAIAAAAA= + AAAAAAAAAAAAAAAEgAAEAAAAAAAAAIAAAAAAAAAAImA= Options\ProducerOptions.cs + + + + ACEAAwgAEgAAQACEQIAAAAAAwAAgAAAAAAAAIAAAABA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAACAAAAAAAACAAAAAAAgAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AACAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEQACAAAABAAEAAAAAAACAAAAAAAE= + Options\GlobalOptions.cs + + + + + + QAAAAAAAAAAMAAAFAAAQCAAAAAAAQBAAQAAAEQAAABA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAACAAAQAkAAAAAAAAAAAAAQAAAAEAAAAAAAI= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAIAEAAAA= + Options\GlobalOptions.cs + + + + + + BAAAAAAAAAAgAAAEAAAAAAAAAAAAAAAAAAAAAAABAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAQAIAAAEAAAAAAAAAAAAIBAAOAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAIACQAACACAAAEEAAAIAAAAEAAAAgAAgAAAAAAQQEA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAIAABEAEAQIgACIAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAIEAAAAAAAAAAAAAAAAAAIAAAAAAEA= + Options\GlobalOptions.cs + + + + + + AAAAAAACAAAAAIAEAAAAAAAAAAAAAAAAABAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + gAAAAAAAAAAAAAAEAAAAAAAAKAAAAAACAQAAAAABAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAQAEAAAAAAAAAAAAAAAAAAQAACAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAQAEAAAAACAEAAACAQABAAAAAAAAAAAAAAAABAA= + Options\GlobalOptions.cs + + + + + + AAQAAAUAAAAAAAAEAAAAAAABAAAAAAABAAAIAAIAQAA= + Options\GlobalOptions.cs + + + + + + QAAAAAAAAAAAAAAAAAAQCAAAAAAAQBAAQAAAEAAAABA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAACAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAQAAAAAAAAAAAAAAAIAAAAAAAAAAAABA= + Options\GlobalOptions.cs + + \ No newline at end of file diff --git a/src/common/Smi.Common/Options/OptionsDecorator.cs b/src/common/Smi.Common/Options/OptionsDecorator.cs new file mode 100644 index 000000000..5ef1d3941 --- /dev/null +++ b/src/common/Smi.Common/Options/OptionsDecorator.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Smi.Common.Options +{ + /// + /// For classes that modify e.g. populate passwords from a vault etc + /// + public interface IOptionsDecorator + { + GlobalOptions Decorate(GlobalOptions options); + } +} diff --git a/src/common/Smi.Common/Options/OptionsFactory.cs b/src/common/Smi.Common/Options/OptionsFactory.cs new file mode 100644 index 000000000..6e02f857c --- /dev/null +++ b/src/common/Smi.Common/Options/OptionsFactory.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using YamlDotNet.Serialization; + +namespace Smi.Common.Options +{ + public class GlobalOptionsFactory + { + public List Decorators {get;set;} + + public GlobalOptionsFactory() + { + + } + public GlobalOptions Load(string environment = "default", string currentDirectory = null, bool decorate = true) + { + IDeserializer deserializer = new DeserializerBuilder() + .WithObjectFactory(GetGlobalOption) + .IgnoreUnmatchedProperties() + .Build(); + + currentDirectory = currentDirectory ?? Environment.CurrentDirectory; + + // Make sure environment ends with yaml + if (!(environment.EndsWith(".yaml") || environment.EndsWith(".yml"))) + environment += ".yaml"; + + // If the yaml file doesn't exist and the path is relative, try looking in currentDirectory instead + if (!File.Exists(environment) && !Path.IsPathRooted(environment)) + environment = Path.Combine(currentDirectory, environment); + + string text = File.ReadAllText(environment); + + var globals = deserializer.Deserialize(new StringReader(text)); + globals.CurrentDirectory = currentDirectory; + globals.MicroserviceOptions = new MicroserviceOptions(); + + return Decorate(globals); + } + + /// + /// Applies all to + /// + /// + /// + private GlobalOptions Decorate(GlobalOptions globals) + { + foreach(var d in Decorators) + globals = d.Decorate(globals); + + return globals; + } + + public GlobalOptions Load(CliOptions cliOptions) + { + //load but do not decorate + GlobalOptions globalOptions = Load(cliOptions.YamlFile,null); + globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); + + return globalOptions; + } + + + private object GetGlobalOption(Type arg) + { + return arg == typeof(GlobalOptions) ? + new GlobalOptions() : + Activator.CreateInstance(arg); + } + } +} diff --git a/src/common/Smi.Common/Options/RabbitMqConfigOptions.md b/src/common/Smi.Common/Options/RabbitMqConfigOptions.md index 2b49fdedb..a20e6ca40 100644 --- a/src/common/Smi.Common/Options/RabbitMqConfigOptions.md +++ b/src/common/Smi.Common/Options/RabbitMqConfigOptions.md @@ -161,10 +161,10 @@ You can then pass the GlobalOptions instance to your specific host in the bootst ```csharp private static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); Parser.Default.ParseArguments(args, cli); - var options = GlobalOptions.Load(cli.YamlFile); + var options = new GlobalOptionsFactory().Load(cli.YamlFile); var bootstrapper = new MicroserviceHostBootstrapper( () => new DicomTagReaderHost(options)); diff --git a/src/common/Smi.Common/README.md b/src/common/Smi.Common/README.md index ba3f9af97..c3d41971f 100644 --- a/src/common/Smi.Common/README.md +++ b/src/common/Smi.Common/README.md @@ -18,7 +18,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); // ... } @@ -35,7 +35,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); // will use the 'default.yaml' file + var options = new GlobalOptionsFactory().Load(); // will use the 'default.yaml' file var consumerOptions = options.MyHostOptions; // you don't really need this here... @@ -111,7 +111,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); var bootstrapper = new MicroserviceHostBootstrapper( () => new CohortPackagerHost(options)); diff --git a/src/microservices/Microservices.CohortExtractor/Program.cs b/src/microservices/Microservices.CohortExtractor/Program.cs index b9080c4e1..3eaa99419 100644 --- a/src/microservices/Microservices.CohortExtractor/Program.cs +++ b/src/microservices/Microservices.CohortExtractor/Program.cs @@ -23,7 +23,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( (a) => { - GlobalOptions options = GlobalOptions.Load(a); + GlobalOptions options = new GlobalOptionsFactory().Load(a); var bootStrapper = new MicroserviceHostBootstrapper(() => new CohortExtractorHost(options, null, null)); //Use the auditor and request fullfilers specified in the yaml diff --git a/src/microservices/Microservices.CohortPackager/Program.cs b/src/microservices/Microservices.CohortPackager/Program.cs index 11135e627..50f60f177 100644 --- a/src/microservices/Microservices.CohortPackager/Program.cs +++ b/src/microservices/Microservices.CohortPackager/Program.cs @@ -22,7 +22,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult((cohortPackagerCliOptions) => { - GlobalOptions globalOptions = GlobalOptions.Load(cohortPackagerCliOptions); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load(cohortPackagerCliOptions); if (cohortPackagerCliOptions.ExtractionId != default) return RecreateReport(globalOptions, cohortPackagerCliOptions.ExtractionId); diff --git a/src/microservices/Microservices.DeadLetterReprocessor/Program.cs b/src/microservices/Microservices.DeadLetterReprocessor/Program.cs index f8fcf2adc..98988dcb7 100644 --- a/src/microservices/Microservices.DeadLetterReprocessor/Program.cs +++ b/src/microservices/Microservices.DeadLetterReprocessor/Program.cs @@ -14,7 +14,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args) .MapResult(deadLetterCliOptions => { - GlobalOptions globals = GlobalOptions.Load(deadLetterCliOptions); + GlobalOptions globals = new GlobalOptionsFactory().Load(deadLetterCliOptions); var bootstrapper = new MicroserviceHostBootstrapper(() => new DeadLetterReprocessorHost(globals, deadLetterCliOptions)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.DicomRelationalMapper/Program.cs b/src/microservices/Microservices.DicomRelationalMapper/Program.cs index d60729d2c..e06ae8c0c 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Program.cs +++ b/src/microservices/Microservices.DicomRelationalMapper/Program.cs @@ -12,7 +12,7 @@ private static int Main(string[] args) { return Parser.Default.ParseArguments(args).MapResult((o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomRelationalMapperHost(options)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.DicomReprocessor/Program.cs b/src/microservices/Microservices.DicomReprocessor/Program.cs index a9359cf3b..07acd012d 100644 --- a/src/microservices/Microservices.DicomReprocessor/Program.cs +++ b/src/microservices/Microservices.DicomReprocessor/Program.cs @@ -19,7 +19,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args) .MapResult(dicomReprocessorCliOptions => { - GlobalOptions options = GlobalOptions.Load(dicomReprocessorCliOptions); + GlobalOptions options = new GlobalOptionsFactory().Load(dicomReprocessorCliOptions); var bootStrapper = new MicroserviceHostBootstrapper(() => new DicomReprocessorHost(options, dicomReprocessorCliOptions)); return bootStrapper.Main(); diff --git a/src/microservices/Microservices.DicomTagReader/Program.cs b/src/microservices/Microservices.DicomTagReader/Program.cs index 7500349b5..a85f9b95f 100644 --- a/src/microservices/Microservices.DicomTagReader/Program.cs +++ b/src/microservices/Microservices.DicomTagReader/Program.cs @@ -17,7 +17,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( (o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomTagReaderHost(options)); diff --git a/src/microservices/Microservices.FileCopier/Program.cs b/src/microservices/Microservices.FileCopier/Program.cs index 46efe138b..1f2bddd69 100644 --- a/src/microservices/Microservices.FileCopier/Program.cs +++ b/src/microservices/Microservices.FileCopier/Program.cs @@ -16,7 +16,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( (o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootstrapper = new MicroserviceHostBootstrapper(() => new FileCopierHost(options)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.IdentifierMapper/Program.cs b/src/microservices/Microservices.IdentifierMapper/Program.cs index e030baa0b..01b16a78e 100644 --- a/src/microservices/Microservices.IdentifierMapper/Program.cs +++ b/src/microservices/Microservices.IdentifierMapper/Program.cs @@ -13,7 +13,7 @@ public static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( cliOptions => { - GlobalOptions options = GlobalOptions.Load(cliOptions); + GlobalOptions options = new GlobalOptionsFactory().Load(cliOptions); var bootstrapper = new MicroserviceHostBootstrapper(() => new IdentifierMapperHost(options)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.IsIdentifiable/Program.cs b/src/microservices/Microservices.IsIdentifiable/Program.cs index 40ca19772..512d262f8 100644 --- a/src/microservices/Microservices.IsIdentifiable/Program.cs +++ b/src/microservices/Microservices.IsIdentifiable/Program.cs @@ -35,7 +35,7 @@ public static int Main(string[] args) //If running as a self contained micro service (getting messages from RabbitMQ) if (args.Length == 1 && string.Equals(args[0], "--service", StringComparison.CurrentCultureIgnoreCase)) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); var bootstrapper = new MicroserviceHostBootstrapper( () => new IsIdentifiableHost(options)); @@ -115,7 +115,7 @@ private static int Run(IsIdentifiableMongoOptions opts) private static int Run(IsIdentifiableServiceOptions opts) { - var options = GlobalOptions.Load(opts.YamlFile); + var options = new GlobalOptionsFactory().Load(opts.YamlFile); var bootstrapper = new MicroserviceHostBootstrapper( () => new IsIdentifiableHost(options)); diff --git a/src/microservices/Microservices.MongoDbPopulator/Program.cs b/src/microservices/Microservices.MongoDbPopulator/Program.cs index fa57aab9e..93f58320d 100644 --- a/src/microservices/Microservices.MongoDbPopulator/Program.cs +++ b/src/microservices/Microservices.MongoDbPopulator/Program.cs @@ -16,7 +16,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult((o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootStrapper = new MicroserviceHostBootstrapper(() => new MongoDbPopulatorHost(options)); return bootStrapper.Main(); diff --git a/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs b/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs index 7c5821e51..211c42a55 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs +++ b/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs @@ -23,7 +23,7 @@ public void OneTimeSetUp() { TestLogger.Setup(); - GlobalOptions globalOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _mongoOptions = globalOptions.MongoDatabases.DicomStoreOptions; } diff --git a/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs b/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs index be38a433f..b1b7ae7e6 100644 --- a/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs +++ b/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs @@ -34,7 +34,7 @@ public class DeadLetterTestHelper : IDisposable public void SetUpSuite() { TestLogger.Setup(); - GlobalOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var testConnectionFactory = new ConnectionFactory { diff --git a/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs b/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs index 9b8aed6dd..ef1af3fda 100644 --- a/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs +++ b/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs @@ -14,7 +14,7 @@ public class HeaderPreservationTest [Test] public void SendHeader() { - var o = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var o = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumerOptions = new ConsumerOptions(); consumerOptions.QueueName = "TEST.HeaderPreservationTest_Read1"; diff --git a/tests/common/Smi.Common.Tests/OptionsTests.cs b/tests/common/Smi.Common.Tests/OptionsTests.cs index 870379e2b..f0f6ed2c2 100644 --- a/tests/common/Smi.Common.Tests/OptionsTests.cs +++ b/tests/common/Smi.Common.Tests/OptionsTests.cs @@ -12,7 +12,7 @@ public class OptionsTests [TestCase("default.yaml")] public void GlobalOptions_Test(string template) { - GlobalOptions globals = GlobalOptions.Load(template, TestContext.CurrentContext.TestDirectory); + GlobalOptions globals = new GlobalOptionsFactory().Load(template, TestContext.CurrentContext.TestDirectory); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RabbitOptions.RabbitMqHostName)); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.FileSystemOptions.FileSystemRoot)); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RDMPOptions.CatalogueConnectionString)); @@ -46,7 +46,7 @@ public void TestVerifyPopulatedChecks() [Test] public void Test_GlobalOptionsUseTestValues_Nulls() { - GlobalOptions g = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions g = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); Assert.IsNotNull(g.RabbitOptions.RabbitMqHostName); g.UseTestValues(null, null, null, null, null); @@ -56,7 +56,7 @@ public void Test_GlobalOptionsUseTestValues_Nulls() [Test] public void Test_GlobalOptions_FileReadOption_ThrowsException() { - GlobalOptions g = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions g = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); g.DicomTagReaderOptions.FileReadOption = "SkipLargeTags"; Assert.Throws(() => g.DicomTagReaderOptions.GetReadOption()); diff --git a/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs b/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs index 1312a7122..d939132a6 100644 --- a/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs +++ b/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs @@ -40,7 +40,7 @@ public void OneTimeSetUp() [SetUp] public void SetUp() { - _testOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _testOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _testProducerOptions = new ProducerOptions { @@ -223,7 +223,7 @@ public void Test_Shutdown(Type consumerType) NLog.Config.SimpleConfigurator.ConfigureForTargetLogging(target, LogLevel.Debug); - var o = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var o = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumer = (IConsumer)Activator.CreateInstance(consumerType); diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs index c778f0a9d..ebace6e1b 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -52,7 +52,7 @@ public void TearDown() { } [Test] public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() { - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.CohortExtractorOptions.ExtractAnonRoutingKey = "anon"; globals.CohortExtractorOptions.ExtractIdentRoutingKey = ""; AssertMessagePublishedWithSpecifiedKey(globals, false, "anon"); @@ -61,7 +61,7 @@ public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() [Test] public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() { - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.CohortExtractorOptions.ExtractAnonRoutingKey = ""; globals.CohortExtractorOptions.ExtractIdentRoutingKey = "ident"; AssertMessagePublishedWithSpecifiedKey(globals, true, "ident"); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs index 398affd16..6a2ae7b0d 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs @@ -116,7 +116,7 @@ public void Test_CohortPackagerHost_HappyPath() }; - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); @@ -210,7 +210,7 @@ public void Test_CohortPackagerHost_IdentifiableExtraction() IsIdentifiableExtraction = true, }; - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs index c1e7e7398..0ca26a4e5 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs @@ -76,7 +76,7 @@ public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages var template = ImageTableTemplateCollection.LoadFrom(_templateXml); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; @@ -158,7 +158,7 @@ public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages) var template = ImageTableTemplateCollection.LoadFrom(_templateXml); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs index 1f1bedbd4..92d3f629f 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs @@ -31,7 +31,7 @@ public void TestCreatingNamer_CorrectType(DatabaseType dbType, string typeName, var cata = Import(tbl); - var globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumerOptions = globals.DicomRelationalMapperOptions; var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs index a74781a2d..eb566f900 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs @@ -34,7 +34,7 @@ public void Setup() { BlitzMainDataTables(); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource)); diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs index a62980c8c..2cb9c258f 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs @@ -57,7 +57,7 @@ public void SetupSuite(DiscoveredDatabase server, bool persistentRaw = false, st { TestLogger.Setup(); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.UseTestValues( RequiresRabbit.GetConnectionFactory(), diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/RunMeFirstTests/RunMeFirstMongoServers.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/RunMeFirstTests/RunMeFirstMongoServers.cs index 5f15c091f..bb91a0136 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/RunMeFirstTests/RunMeFirstMongoServers.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/RunMeFirstTests/RunMeFirstMongoServers.cs @@ -19,7 +19,7 @@ public void TestMongoAvailable() [Test, RequiresRabbit] public void RabbitAvailable() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var rabbitOptions = options.RabbitOptions; Console.WriteLine("Checking the following configuration:" + Environment.NewLine + rabbitOptions); diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs b/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs index 571aade3d..fd2808608 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs +++ b/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs @@ -71,7 +71,7 @@ public void ResetSuite() private void SetUpDefaults() { - Options = GlobalOptions.Load("default", TestContext.CurrentContext.TestDirectory); + Options = new GlobalOptionsFactory().Load("default", TestContext.CurrentContext.TestDirectory); AccessionConsumerOptions = Options.DicomTagReaderOptions; diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs index 0049a1044..0fe1bf683 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -44,7 +44,7 @@ public void TearDown() { } [Test] public void Test_FileCopierHost_HappyPath() { - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.FileSystemOptions.FileSystemRoot = "root"; globals.FileSystemOptions.ExtractRoot = "exroot"; diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs b/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs index 7cde5e278..52ae6a455 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs @@ -106,7 +106,7 @@ public void TestIdentifierSwap_NoCache(DatabaseType type, Test test) public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom) { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var mappingDataTable = new DataTable("IdMap"); mappingDataTable.Columns.Add("priv"); @@ -535,7 +535,7 @@ public void TestSwapCache() DiscoveredDatabase db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); - GlobalOptions options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IdentifierMapperOptions = new IdentifierMapperOptions { MappingConnectionString = db.Server.Builder.ConnectionString, diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs index 891602121..03417f0d1 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs @@ -25,7 +25,7 @@ public void OneTimeSetUp() [Test] public void TestClassifierName_NoClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IsIdentifiableOptions.ClassifierType = ""; var ex = Assert.Throws(() => new IsIdentifiableHost(options, false)); @@ -35,7 +35,7 @@ public void TestClassifierName_NoClassifier() [Test] public void TestClassifierName_NotRecognized() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.WorkDirectory; options.IsIdentifiableOptions.ClassifierType = "HappyFunTimes"; @@ -46,7 +46,7 @@ public void TestClassifierName_NotRecognized() [Test] public void TestClassifierName_ValidClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm"); TestData.Create(testDcm); @@ -80,7 +80,7 @@ public void TestClassifierName_ValidClassifier() [Test] public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); // Create a test data directory containing IsIdentifiableRules with 0 rules, and tessdata with the eng.traineddata classifier // TODO(rkm 2020-04-14) This is a stop-gap solution until the tests are properly refactored diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs b/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs index 6a82b7b12..ea6afbd5e 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs @@ -31,7 +31,7 @@ public void OneTimeSetUp() [SetUp] public void SetUp() { - _testOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _testOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); } /// diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs b/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs index fe13a03a0..4413646bd 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs @@ -78,7 +78,7 @@ public void SetupSuite() public static GlobalOptions GetNewMongoDbPopulatorOptions() { - GlobalOptions options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.MongoDatabases.DicomStoreOptions.DatabaseName = TestDbName; options.MongoDbPopulatorOptions.MongoDbFlushTime = 1; //1 second From 3397e594edb79aa2dbe14d6dc79a28a72fd2542b Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 10:39:16 +0100 Subject: [PATCH 113/138] Implemented decorator pattern for environment variables --- CHANGELOG.md | 2 +- .../Smi.Common/Options/ConsumerOptions.cs | 2 +- .../Options/EnvironmentVariableDecorator.cs | 37 +++++++++++++++++++ .../Smi.Common/Options/GlobalOptions.cs | 31 +++++++++------- .../Smi.Common/Options/IOptionsDecorator.cs | 14 +++++++ .../Smi.Common/Options/OptionsDecorator.cs | 35 ++++++++++++++---- .../Smi.Common/Options/OptionsFactory.cs | 6 +-- tests/common/Smi.Common.Tests/OptionsTests.cs | 20 ++++++++++ 8 files changed, 122 insertions(+), 25 deletions(-) create mode 100644 src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs create mode 100644 src/common/Smi.Common/Options/IOptionsDecorator.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d4d30ef..09154bb62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed -- Environment variables are no longer used. Previous settings now appear in configuration file +- Environment variables are no longer required. Previous settings now appear in configuration file - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` - Removed `ISIDENTIFIABLE_NUMTHREADS` as it didn't work correctly anyway diff --git a/src/common/Smi.Common/Options/ConsumerOptions.cs b/src/common/Smi.Common/Options/ConsumerOptions.cs index 74d317e20..825660632 100644 --- a/src/common/Smi.Common/Options/ConsumerOptions.cs +++ b/src/common/Smi.Common/Options/ConsumerOptions.cs @@ -5,7 +5,7 @@ namespace Smi.Common.Options /// /// Configuration options needed to receive messages from a RabbitMQ queue. /// - public class ConsumerOptions + public class ConsumerOptions : IOptions { /// /// Name of the queue to consume from. diff --git a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs new file mode 100644 index 000000000..633862d7b --- /dev/null +++ b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Smi.Common.Options +{ + + /// + /// Populates values in based on environment variables + /// + public class EnvironmentVariableDecorator : OptionsDecorator + { + public override GlobalOptions Decorate(GlobalOptions options) + { + ForAll(options,SetMongoPassword); + + var logsRoot = Environment.GetEnvironmentVariable("SMI_LOGS_ROOT"); + + if(!string.IsNullOrWhiteSpace(logsRoot)) + options.LogsRoot = logsRoot; + + return options; + } + + private MongoDbOptions SetMongoPassword(MongoDbOptions opt) + { + //get the environment variables current value + var envVar = Environment.GetEnvironmentVariable("MONGO_SERVICE_PASSWORD"); + + //if theres an env var for it and there are mongodb options being used + if(!string.IsNullOrWhiteSpace(envVar) && opt != null) + opt.Password = envVar; + + return opt; + } + } +} diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 8fba1d990..bf0804a96 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -18,7 +18,12 @@ namespace Smi.Common.Options { - public class GlobalOptions + public interface IOptions + { + + } + + public class GlobalOptions : IOptions { #region AllOptions @@ -88,7 +93,7 @@ public class IsIdentifiableOptions : ConsumerOptions } [UsedImplicitly] - public class MicroserviceOptions + public class MicroserviceOptions : IOptions { public bool TraceLogging { get; set; } = true; @@ -106,7 +111,7 @@ public override string ToString() } [UsedImplicitly] - public class ProcessDirectoryOptions + public class ProcessDirectoryOptions : IOptions { public ProducerOptions AccessionDirectoryProducerOptions { get; set; } @@ -117,7 +122,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDbPopulatorOptions + public class MongoDbPopulatorOptions : IOptions { public ConsumerOptions SeriesQueueConsumerOptions { get; set; } public ConsumerOptions ImageQueueConsumerOptions { get; set; } @@ -191,7 +196,7 @@ public IMappingTableOptions Clone() } } - public interface IMappingTableOptions + public interface IMappingTableOptions : IOptions { string MappingConnectionString { get; } string MappingTableName { get; set; } @@ -261,7 +266,7 @@ public enum TagProcessorMode } [UsedImplicitly] - public class DicomReprocessorOptions + public class DicomReprocessorOptions : IOptions { public ProcessingMode ProcessingMode { get; set; } @@ -294,7 +299,7 @@ public enum ProcessingMode } [UsedImplicitly] - public class CohortPackagerOptions + public class CohortPackagerOptions : IOptions { public ConsumerOptions ExtractRequestInfoOptions { get; set; } public ConsumerOptions FileCollectionInfoOptions { get; set; } @@ -406,7 +411,7 @@ public override string ToString() } [UsedImplicitly] - public class DeadLetterReprocessorOptions + public class DeadLetterReprocessorOptions : IOptions { public ConsumerOptions DeadLetterConsumerOptions { get; set; } @@ -421,7 +426,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDatabases + public class MongoDatabases : IOptions { public MongoDbOptions DicomStoreOptions { get; set; } @@ -436,7 +441,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDbOptions + public class MongoDbOptions : IOptions { public string HostName { get; set; } = "localhost"; public int Port { get; set; } = 27017; @@ -467,7 +472,7 @@ public override string ToString() /// Describes the location of the Microsoft Sql Server RDMP platform databases which keep track of load configurations, available datasets (tables) etc /// [UsedImplicitly] - public class RDMPOptions + public class RDMPOptions : IOptions { public string CatalogueConnectionString { get; set; } public string DataExportConnectionString { get; set; } @@ -492,7 +497,7 @@ public override string ToString() /// Describes the root location of all images, file names should be expressed as relative paths (relative to this root). /// [UsedImplicitly] - public class FileSystemOptions + public class FileSystemOptions : IOptions { /// /// If set, services will require that is set and points to a valid directory. @@ -529,7 +534,7 @@ public override string ToString() /// /// Describes the location of the rabbit server for sending messages to /// - public class RabbitOptions + public class RabbitOptions : IOptions { public string RabbitMqHostName { get; set; } public int RabbitMqHostPort { get; set; } diff --git a/src/common/Smi.Common/Options/IOptionsDecorator.cs b/src/common/Smi.Common/Options/IOptionsDecorator.cs new file mode 100644 index 000000000..5ef1d3941 --- /dev/null +++ b/src/common/Smi.Common/Options/IOptionsDecorator.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Smi.Common.Options +{ + /// + /// For classes that modify e.g. populate passwords from a vault etc + /// + public interface IOptionsDecorator + { + GlobalOptions Decorate(GlobalOptions options); + } +} diff --git a/src/common/Smi.Common/Options/OptionsDecorator.cs b/src/common/Smi.Common/Options/OptionsDecorator.cs index 5ef1d3941..84060e098 100644 --- a/src/common/Smi.Common/Options/OptionsDecorator.cs +++ b/src/common/Smi.Common/Options/OptionsDecorator.cs @@ -1,14 +1,35 @@ using System; -using System.Collections.Generic; -using System.Text; +using System.Reflection; namespace Smi.Common.Options { - /// - /// For classes that modify e.g. populate passwords from a vault etc - /// - public interface IOptionsDecorator + public abstract class OptionsDecorator : IOptionsDecorator { - GlobalOptions Decorate(GlobalOptions options); + public abstract GlobalOptions Decorate(GlobalOptions options); + + protected void ForAll(IOptions globals,Func setter) where T: IOptions + { + //for each property on branch + foreach(PropertyInfo p in globals.GetType().GetProperties()) + { + var currentValue = p.GetValue(globals); + + //if it's a T then call the action (note that we check the property Type because we are interested in the property even if it is null + if(p.PropertyType.IsAssignableFrom(typeof(T))) + { + //the delegate changes the value of the property of Type T (or creates a new instance from scratch) + var result = setter((T)currentValue); + + //store the result of the delegate for this property + p.SetValue(globals,result); + } + + //process it's children + if(currentValue is IOptions subOptions) + { + ForAll(subOptions,setter); + } + } + } } } diff --git a/src/common/Smi.Common/Options/OptionsFactory.cs b/src/common/Smi.Common/Options/OptionsFactory.cs index 6e02f857c..4bb0e3e56 100644 --- a/src/common/Smi.Common/Options/OptionsFactory.cs +++ b/src/common/Smi.Common/Options/OptionsFactory.cs @@ -8,13 +8,13 @@ namespace Smi.Common.Options { public class GlobalOptionsFactory { - public List Decorators {get;set;} + public List Decorators {get;set;} = new List(); public GlobalOptionsFactory() { - + Decorators.Add(new EnvironmentVariableDecorator()); } - public GlobalOptions Load(string environment = "default", string currentDirectory = null, bool decorate = true) + public GlobalOptions Load(string environment = "default", string currentDirectory = null) { IDeserializer deserializer = new DeserializerBuilder() .WithObjectFactory(GetGlobalOption) diff --git a/tests/common/Smi.Common.Tests/OptionsTests.cs b/tests/common/Smi.Common.Tests/OptionsTests.cs index f0f6ed2c2..c98246db6 100644 --- a/tests/common/Smi.Common.Tests/OptionsTests.cs +++ b/tests/common/Smi.Common.Tests/OptionsTests.cs @@ -61,5 +61,25 @@ public void Test_GlobalOptions_FileReadOption_ThrowsException() Assert.Throws(() => g.DicomTagReaderOptions.GetReadOption()); } + + + private class TestDecorator : OptionsDecorator + { + public override GlobalOptions Decorate(GlobalOptions options) + { + ForAll(options,(o)=> new MongoDbOptions{DatabaseName = "FFFFF" }); + return options; + } + } + + [Test] + public void TestDecorators() + { + var factory = new GlobalOptionsFactory(){ Decorators = { new TestDecorator()} }; + var g = factory.Load(); + Assert.AreEqual("FFFFF",g.MongoDatabases.DeadLetterStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF",g.MongoDatabases.DicomStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF",g.MongoDatabases.ExtractionStoreOptions.DatabaseName); + } } } From 7914d167713d9717192a946dc543c404a9576483 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 10:54:08 +0100 Subject: [PATCH 114/138] Fixed producer not being IOptions --- .../Smi.Common/Options/GlobalOptions.cs | 2 +- .../Options/Microservices.Common.Options.cd | 104 ++++++++++-------- .../Smi.Common/Options/ProducerOptions.cs | 2 +- 3 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index bf0804a96..9c502f0e6 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -142,7 +142,7 @@ public override string ToString() } [UsedImplicitly] - public class IdentifierMapperOptions : ConsumerOptions, IMappingTableOptions + public class IdentifierMapperOptions : ConsumerOptions { public ProducerOptions AnonImagesProducerOptions { get; set; } public string MappingConnectionString { get; set; } diff --git a/src/common/Smi.Common/Options/Microservices.Common.Options.cd b/src/common/Smi.Common/Options/Microservices.Common.Options.cd index d6f8c7dcf..dd711fbd3 100644 --- a/src/common/Smi.Common/Options/Microservices.Common.Options.cd +++ b/src/common/Smi.Common/Options/Microservices.Common.Options.cd @@ -1,172 +1,182 @@  - - - - AAAAAAAAAAAAEgAAAAAAQQAAAAAQAAAAAAACAIAAAAA= - Options\MicroservicesOptions.cs - - - - - - + AAAAAEAAAAAAAAAEAAAAAAAAAAAAAQAAAAgAAAAAIAA= Options\ConsumerOptions.cs + - + - AAAAAAAAAAAAAAAEgAAEAAAAAAAAAIAAAAAAAAAAImA= + AAAAAAAAAAAAAAAEAAAEAAAAAAAAAIAAAAAAAAAAImA= Options\ProducerOptions.cs + - - + + ACEAAwgAEgAAQACEQIAAAAAAwAAgAAAAAAAAIAAAABA= Options\GlobalOptions.cs + - + AAAAAAAAAACAAAAAAAACAAAAAAAgAAAAAAAAAAAAAAA= Options\GlobalOptions.cs - - + + AAAAAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAA= Options\GlobalOptions.cs + - - + + AACAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAA= Options\GlobalOptions.cs + - - + + AAAAAAAAAAAAAAAEQACAAAABAAEAAAAAAACAAAAAAAE= Options\GlobalOptions.cs + - + QAAAAAAAAAAMAAAFAAAQCAAAAAAAQBAAQAAAEQAAABA= Options\GlobalOptions.cs - - + AAAAAAAAACAAAQAkAAAAAAAAAAAAAQAAAAEAAAAAAAI= Options\GlobalOptions.cs - + AAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAIAEAAAA= Options\GlobalOptions.cs - - + + BAAAAAAAAAAgAAAEAAAAAAAAAAAAAAAAAAAAAAABAAA= Options\GlobalOptions.cs + - - + + AAAAAAAAAQAIAAAEAAAAAAAAAAAAIBAAOAAAAAAAAAA= Options\GlobalOptions.cs + - + AAIACQAACACAAAEEAAAIAAAAEAAAAgAAgAAAAAAQQEA= Options\GlobalOptions.cs - + AAAAAAAAAAAAAAAEAAAAAAAAAIAABEAEAQIgACIAAAA= Options\GlobalOptions.cs - - + + AAAAAAAAAAAAAAIEAAAAAAAAAAAAAAAAAAIAAAAAAEA= Options\GlobalOptions.cs + - - + + AAAAAAACAAAAAIAEAAAAAAAAAAAAAAAAABAAAAAAAAA= Options\GlobalOptions.cs + - - + + gAAAAAAAAAAAAAAEAAAAAAAAKAAAAAACAQAAAAABAAA= Options\GlobalOptions.cs + - - + + AAAAAAAAAAAAAQAEAAAAAAAAAAAAAAAAAAQAACAAAAA= Options\GlobalOptions.cs + - - + + AAAAAQAEAAAAACAEAAACAQABAAAAAAAAAAAAAAAABAA= Options\GlobalOptions.cs + - - + + AAQAAAUAAAAAAAAEAAAAAAABAAAAAAABAAAIAAIAQAA= Options\GlobalOptions.cs + - + QAAAAAAAAAAAAAAAAAAQCAAAAAAAQBAAQAAAEAAAABA= Options\GlobalOptions.cs + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + - + AAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAACAAAAA= Options\GlobalOptions.cs - + AAAAAAAAAAAAQAAAAAAAAAAAAAAAIAAAAAAAAAAAABA= Options\GlobalOptions.cs diff --git a/src/common/Smi.Common/Options/ProducerOptions.cs b/src/common/Smi.Common/Options/ProducerOptions.cs index fb4dc11ed..51ee78630 100644 --- a/src/common/Smi.Common/Options/ProducerOptions.cs +++ b/src/common/Smi.Common/Options/ProducerOptions.cs @@ -4,7 +4,7 @@ namespace Smi.Common.Options /// /// Configuration options needed to send messages to a RabbitMQ exchange /// - public class ProducerOptions + public class ProducerOptions : IOptions { /// /// Name of the RabbitMQ exchange to send messages to From f2adc8fb3c64bdc8195a3709add11900a3b8fc6f Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Wed, 9 Sep 2020 11:03:58 +0100 Subject: [PATCH 115/138] Have Appveyor quit as early as possible if not needed --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 888ba7179..c60232208 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,6 @@ version: 1.0.{build} init: + - cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - git config --global core.autocrlf true image: Visual Studio 2019 services: @@ -11,7 +12,6 @@ cache: - '%USERPROFILE%\.nuget\packages -> **\*.csproj' before_build: -- cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" - cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.initiate())\"" From de6f01f354fedf8566bdff9d4df6f18bc733ddb4 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 11:09:40 +0100 Subject: [PATCH 116/138] Fixed missing interface --- src/common/Smi.Common/Options/GlobalOptions.cs | 2 +- src/common/Smi.Common/Options/Microservices.Common.Options.cd | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 9c502f0e6..bf0804a96 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -142,7 +142,7 @@ public override string ToString() } [UsedImplicitly] - public class IdentifierMapperOptions : ConsumerOptions + public class IdentifierMapperOptions : ConsumerOptions, IMappingTableOptions { public ProducerOptions AnonImagesProducerOptions { get; set; } public string MappingConnectionString { get; set; } diff --git a/src/common/Smi.Common/Options/Microservices.Common.Options.cd b/src/common/Smi.Common/Options/Microservices.Common.Options.cd index dd711fbd3..cdd71132b 100644 --- a/src/common/Smi.Common/Options/Microservices.Common.Options.cd +++ b/src/common/Smi.Common/Options/Microservices.Common.Options.cd @@ -61,6 +61,7 @@ QAAAAAAAAAAMAAAFAAAQCAAAAAAAQBAAQAAAEQAAABA= Options\GlobalOptions.cs + From 9c6d7d54aafe0620ac52ebeb0dc528035acc94ec Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 11:49:16 +0100 Subject: [PATCH 117/138] Added new rejector that queries a column for patients to reject --- CHANGELOG.md | 1 + .../Smi.Common/Options/GlobalOptions.cs | 5 ++ .../Execution/CohortExtractorHost.cs | 3 + .../RequestFulfillers/PatientRejector.cs | 75 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 28da3964f..5e7aa2027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions - Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool +- Added new rejector that throws out patients whose IDs are stored in a database table. Set `RejectPatientsIn` option in yaml to enable this ## [1.11.1] - 2020-08-12 diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 2fcc49a31..1c93e1f59 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -403,6 +403,11 @@ public string AuditorType public ProducerOptions ExtractFilesProducerOptions { get; set; } public ProducerOptions ExtractFilesInfoProducerOptions { get; set; } + + /// + /// ID of a ColumnInfo that contains a list of patients who should not have data extracted for them. e.g. opt out + /// + public int? RejectPatientsIn { get; set; } public override string ToString() { diff --git a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs index c927390f6..bd03fa30b 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs @@ -132,6 +132,9 @@ private void InitializeExtractionSources(IRDMPPlatformRepositoryServiceLocator r if(!string.IsNullOrWhiteSpace(_consumerOptions.RejectorType)) _fulfiller.Rejectors.Add(ObjectFactory.CreateInstance(_consumerOptions.RejectorType,typeof(IRejector).Assembly)); + if(_consumerOptions.RejectPatientsIn.HasValue) + _fulfiller.Rejectors.Add(new PatientRejector(repositoryLocator.CatalogueRepository.GetObjectByID(_consumerOptions.RejectPatientsIn.Value))); + if(_consumerOptions.Blacklists != null) foreach (int id in _consumerOptions.Blacklists) { diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs new file mode 100644 index 000000000..77ccbe389 --- /dev/null +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs @@ -0,0 +1,75 @@ +using MapsDirectlyToDatabaseTable; +using NLog; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.QueryBuilding; +using Rdmp.Core.Repositories; +using ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Text; + +namespace Microservices.CohortExtractor.Execution.RequestFulfillers +{ + class PatientRejector : IRejector + { + HashSet RejectPatients = new HashSet(); + private Logger _logger; + + public PatientRejector(ColumnInfo patientIdColumn) + { + _logger = LogManager.GetCurrentClassLogger(); + + var qb = new QueryBuilder(null,null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(),patientIdColumn)); + + var sql = qb.SQL; + _logger.Info("Running PatientID fetch SQL:" + sql); + + var server = patientIdColumn.TableInfo.Discover(DataAccessContext.DataExport); + + using(var con = server.Database.Server.GetConnection()) + { + var cmd = server.GetCommand(sql,con); + var r = cmd.ExecuteReader(); + + while(r.Read()) + RejectPatients.Add(r[0].ToString()); + } + + _logger.Info($"Found {RejectPatients.Count} patients in the reject list"); + } + + + public bool Reject(DbDataReader row, out string reason) + { + string patientId; + + try + { + //we don't know + if(row["PatientID"] == DBNull.Value) + { + reason = null; + return false; + } + + patientId = (string)row["PatientID"]; + } + catch (Exception ex) + { + throw new Exception("An error occurred determining the PatientID of the record(s) being extracted",ex); + } + + if(RejectPatients.Contains(patientId)) + { + reason = "Patient was in reject list"; + return true; + } + + + reason = null; + return false; + } + } +} From 64fc4536614db52922432d723027c191e114ef8f Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 12:01:30 +0100 Subject: [PATCH 118/138] Added tests --- .../RequestFulfillers/PatientRejector.cs | 4 +- .../PatientRejectorTests.cs | 64 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs index 77ccbe389..b91ff1842 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs @@ -11,9 +11,9 @@ namespace Microservices.CohortExtractor.Execution.RequestFulfillers { - class PatientRejector : IRejector + public class PatientRejector : IRejector { - HashSet RejectPatients = new HashSet(); + HashSet RejectPatients = new HashSet(StringComparer.CurrentCultureIgnoreCase); private Logger _logger; public PatientRejector(ColumnInfo patientIdColumn) diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs new file mode 100644 index 000000000..9d172b2ff --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs @@ -0,0 +1,64 @@ +using FAnsi; +using FAnsi.Discovery; +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Moq; +using NUnit.Framework; +using Rdmp.Core.Curation; +using Rdmp.Core.Curation.Data; +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Text; +using Tests.Common; + +namespace Microservices.CohortExtractor.Tests +{ + class PatientRejectorTests : DatabaseTests + { + [TestCase(DatabaseType.MicrosoftSQLServer)] + [TestCase(DatabaseType.MySql)] + public void TestRejectPatients(DatabaseType type) + { + var server = GetCleanedServer(type); + + var tbl = server.CreateTable("BadPatients",new []{new DatabaseColumnRequest("Pat","varchar(100)") }); + + tbl.Insert(new Dictionary(){{"Pat","Frank"} }); + tbl.Insert(new Dictionary(){{"Pat","Peter" } }); + tbl.Insert(new Dictionary(){{"Pat","Frank" } }); //duplication for the lols + tbl.Insert(new Dictionary(){{"Pat","David" } }); + + new TableInfoImporter(CatalogueRepository,tbl).DoImport(out var ti,out var cols); + + var rejector = new PatientRejector(cols[0]); + + var moqDave = new Mock(); + moqDave.Setup(x => x["PatientId"]) + .Returns("Dave"); + + Assert.IsFalse(rejector.Reject(moqDave.Object,out string reason)); + Assert.IsNull(reason); + + + var moqFrank = new Mock(); + moqFrank.Setup(x => x["PatientId"]) + .Returns("Frank"); + + Assert.IsTrue(rejector.Reject(moqFrank.Object,out reason)); + Assert.AreEqual("Patient was in reject list",reason); + + var moqLowerCaseFrank = new Mock(); + moqLowerCaseFrank.Setup(x => x["PatientId"]) + .Returns("frank"); + + Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object,out reason)); + Assert.AreEqual("Patient was in reject list",reason); + + + + + + } + } +} From d61de5e75dee31159e284b256e5250a97f85eb57 Mon Sep 17 00:00:00 2001 From: tznind Date: Wed, 9 Sep 2020 12:36:36 +0100 Subject: [PATCH 119/138] Fixed not opening connection and PatientId field --- data/microserviceConfigs/default.yaml | 4 ++++ .../Execution/RequestFulfillers/PatientRejector.cs | 9 ++++++--- .../PatientRejectorTests.cs | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index d7f004e3e..a301d277b 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -63,6 +63,10 @@ CohortExtractorOptions: AutoAck: false AllCatalogues: true OnlyCatalogues: [1,2,3] + + # Set this to the ID of a ColumnInfo that contains patient IDs you want forbidden + #RejectPatientsIn: 1234 + # List of IDs of Catalogues to extract from (in ascending order). # Ignored if "AllCatalogues == true" # - 2 diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs index b91ff1842..383d8bd90 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs @@ -15,10 +15,12 @@ public class PatientRejector : IRejector { HashSet RejectPatients = new HashSet(StringComparer.CurrentCultureIgnoreCase); private Logger _logger; + private string _patientIdColumnName; public PatientRejector(ColumnInfo patientIdColumn) { _logger = LogManager.GetCurrentClassLogger(); + _patientIdColumnName = patientIdColumn.GetRuntimeName(); var qb = new QueryBuilder(null,null); qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(),patientIdColumn)); @@ -30,6 +32,7 @@ public PatientRejector(ColumnInfo patientIdColumn) using(var con = server.Database.Server.GetConnection()) { + con.Open(); var cmd = server.GetCommand(sql,con); var r = cmd.ExecuteReader(); @@ -48,17 +51,17 @@ public bool Reject(DbDataReader row, out string reason) try { //we don't know - if(row["PatientID"] == DBNull.Value) + if(row[_patientIdColumnName] == DBNull.Value) { reason = null; return false; } - patientId = (string)row["PatientID"]; + patientId = (string)row[_patientIdColumnName]; } catch (Exception ex) { - throw new Exception("An error occurred determining the PatientID of the record(s) being extracted",ex); + throw new Exception($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_patientIdColumnName}",ex); } if(RejectPatients.Contains(patientId)) diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs index 9d172b2ff..af850442a 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs @@ -34,7 +34,7 @@ public void TestRejectPatients(DatabaseType type) var rejector = new PatientRejector(cols[0]); var moqDave = new Mock(); - moqDave.Setup(x => x["PatientId"]) + moqDave.Setup(x => x["Pat"]) .Returns("Dave"); Assert.IsFalse(rejector.Reject(moqDave.Object,out string reason)); @@ -42,14 +42,14 @@ public void TestRejectPatients(DatabaseType type) var moqFrank = new Mock(); - moqFrank.Setup(x => x["PatientId"]) + moqFrank.Setup(x => x["Pat"]) .Returns("Frank"); Assert.IsTrue(rejector.Reject(moqFrank.Object,out reason)); Assert.AreEqual("Patient was in reject list",reason); var moqLowerCaseFrank = new Mock(); - moqLowerCaseFrank.Setup(x => x["PatientId"]) + moqLowerCaseFrank.Setup(x => x["Pat"]) .Returns("frank"); Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object,out reason)); From 1dcd89d02c73754d6244684008ce973212aa834c Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Wed, 9 Sep 2020 15:55:15 +0100 Subject: [PATCH 120/138] Tidy a bit, add a new test to get coverage over the exception case --- .../RequestFulfillers/PatientRejector.cs | 49 ++++++++-------- .../PatientRejectorTests.cs | 57 ++++++++++++------- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs index 383d8bd90..efafa5781 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs @@ -1,46 +1,46 @@ -using MapsDirectlyToDatabaseTable; +using FAnsi.Discovery; +using MapsDirectlyToDatabaseTable; using NLog; using Rdmp.Core.Curation.Data; using Rdmp.Core.QueryBuilding; -using Rdmp.Core.Repositories; using ReusableLibraryCode.DataAccess; using System; using System.Collections.Generic; using System.Data.Common; -using System.Text; + namespace Microservices.CohortExtractor.Execution.RequestFulfillers { public class PatientRejector : IRejector { - HashSet RejectPatients = new HashSet(StringComparer.CurrentCultureIgnoreCase); - private Logger _logger; - private string _patientIdColumnName; + private readonly HashSet _rejectPatients = new HashSet(StringComparer.CurrentCultureIgnoreCase); + private readonly Logger _logger; + private readonly string _patientIdColumnName; public PatientRejector(ColumnInfo patientIdColumn) { _logger = LogManager.GetCurrentClassLogger(); _patientIdColumnName = patientIdColumn.GetRuntimeName(); - var qb = new QueryBuilder(null,null); - qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(),patientIdColumn)); - - var sql = qb.SQL; + var qb = new QueryBuilder(limitationSQL: null, hashingAlgorithm: null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), patientIdColumn)); + + string sql = qb.SQL; _logger.Info("Running PatientID fetch SQL:" + sql); - var server = patientIdColumn.TableInfo.Discover(DataAccessContext.DataExport); + DiscoveredTable server = patientIdColumn.TableInfo.Discover(DataAccessContext.DataExport); - using(var con = server.Database.Server.GetConnection()) + using (DbConnection con = server.Database.Server.GetConnection()) { con.Open(); - var cmd = server.GetCommand(sql,con); - var r = cmd.ExecuteReader(); + DbCommand cmd = server.GetCommand(sql, con); + DbDataReader reader = cmd.ExecuteReader(); - while(r.Read()) - RejectPatients.Add(r[0].ToString()); + while (reader.Read()) + _rejectPatients.Add(reader[0].ToString()); } - - _logger.Info($"Found {RejectPatients.Count} patients in the reject list"); + + _logger.Info($"Found {_rejectPatients.Count} patients in the reject list"); } @@ -50,8 +50,8 @@ public bool Reject(DbDataReader row, out string reason) try { - //we don't know - if(row[_patientIdColumnName] == DBNull.Value) + // The patient ID is null + if (row[_patientIdColumnName] == DBNull.Value) { reason = null; return false; @@ -59,17 +59,16 @@ public bool Reject(DbDataReader row, out string reason) patientId = (string)row[_patientIdColumnName]; } - catch (Exception ex) + catch (IndexOutOfRangeException ex) { - throw new Exception($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_patientIdColumnName}",ex); + throw new IndexOutOfRangeException($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_patientIdColumnName}", ex); } - - if(RejectPatients.Contains(patientId)) + + if (_rejectPatients.Contains(patientId)) { reason = "Patient was in reject list"; return true; } - reason = null; return false; diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs index af850442a..6fc251277 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs @@ -7,58 +7,71 @@ using Rdmp.Core.Curation.Data; using System; using System.Collections.Generic; -using System.Data; using System.Data.Common; -using System.Text; using Tests.Common; namespace Microservices.CohortExtractor.Tests { - class PatientRejectorTests : DatabaseTests + public class PatientRejectorTests : DatabaseTests { + private const string PatColName = "PatientID"; + [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] - public void TestRejectPatients(DatabaseType type) + public void Test_PatientRejector(DatabaseType type) { - var server = GetCleanedServer(type); + DiscoveredDatabase server = GetCleanedServer(type); + DiscoveredTable tbl = server.CreateTable("BadPatients", new[] { new DatabaseColumnRequest(PatColName, "varchar(100)") }); + + tbl.Insert(new Dictionary { { PatColName, "Frank" } }); + tbl.Insert(new Dictionary { { PatColName, "Peter" } }); + tbl.Insert(new Dictionary { { PatColName, "Frank" } }); //duplication for the lols + tbl.Insert(new Dictionary { { PatColName, "David" } }); - var tbl = server.CreateTable("BadPatients",new []{new DatabaseColumnRequest("Pat","varchar(100)") }); - - tbl.Insert(new Dictionary(){{"Pat","Frank"} }); - tbl.Insert(new Dictionary(){{"Pat","Peter" } }); - tbl.Insert(new Dictionary(){{"Pat","Frank" } }); //duplication for the lols - tbl.Insert(new Dictionary(){{"Pat","David" } }); + new TableInfoImporter(CatalogueRepository, tbl).DoImport(out TableInfo _, out ColumnInfo[] cols); - new TableInfoImporter(CatalogueRepository,tbl).DoImport(out var ti,out var cols); - var rejector = new PatientRejector(cols[0]); var moqDave = new Mock(); - moqDave.Setup(x => x["Pat"]) + moqDave.Setup(x => x[PatColName]) .Returns("Dave"); - Assert.IsFalse(rejector.Reject(moqDave.Object,out string reason)); + Assert.IsFalse(rejector.Reject(moqDave.Object, out string reason)); Assert.IsNull(reason); - var moqFrank = new Mock(); - moqFrank.Setup(x => x["Pat"]) + moqFrank.Setup(x => x[PatColName]) .Returns("Frank"); - Assert.IsTrue(rejector.Reject(moqFrank.Object,out reason)); - Assert.AreEqual("Patient was in reject list",reason); + Assert.IsTrue(rejector.Reject(moqFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); var moqLowerCaseFrank = new Mock(); - moqLowerCaseFrank.Setup(x => x["Pat"]) + moqLowerCaseFrank.Setup(x => x[PatColName]) .Returns("frank"); - Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object,out reason)); - Assert.AreEqual("Patient was in reject list",reason); + Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + } + [TestCase(DatabaseType.MicrosoftSQLServer)] + [TestCase(DatabaseType.MySql)] + public void Test_PatientRejector_MissingColumn_Throws(DatabaseType type) + { + DiscoveredDatabase server = GetCleanedServer(type); + DiscoveredTable tbl = server.CreateTable("BadPatients", new[] { new DatabaseColumnRequest(PatColName, "varchar(100)") }); + new TableInfoImporter(CatalogueRepository, tbl).DoImport(out TableInfo _, out ColumnInfo[] cols); + var rejector = new PatientRejector(cols[0]); + var moqDave = new Mock(); + moqDave + .Setup(x => x[PatColName]) + .Throws(); + var exc = Assert.Throws(() => rejector.Reject(moqDave.Object, out string _)); + Assert.True(exc.Message.Contains($"Expected a column called {PatColName}")); } } } From 7d48a80edc882c7e55c977570679abd9d5e2515e Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 10 Sep 2020 09:38:34 +0100 Subject: [PATCH 121/138] Renamed and split up patient rejector --- .../Execution/CohortExtractorHost.cs | 2 +- .../ColumnInfoValuesRejector.cs | 51 ++++++++++++ .../RequestFulfillers/ColumnValuesRejector.cs | 50 ++++++++++++ .../RequestFulfillers/PatientRejector.cs | 77 ------------------- ...ts.cs => ColumnInfoValuesRejectorTests.cs} | 27 +------ .../ColumnValuesRejectorTests.cs | 55 +++++++++++++ 6 files changed, 161 insertions(+), 101 deletions(-) create mode 100644 src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs create mode 100644 src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs delete mode 100644 src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs rename tests/microservices/Microservices.CohortExtractor.Tests/{PatientRejectorTests.cs => ColumnInfoValuesRejectorTests.cs} (64%) create mode 100644 tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs diff --git a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs index bd03fa30b..781aa26c1 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs @@ -133,7 +133,7 @@ private void InitializeExtractionSources(IRDMPPlatformRepositoryServiceLocator r _fulfiller.Rejectors.Add(ObjectFactory.CreateInstance(_consumerOptions.RejectorType,typeof(IRejector).Assembly)); if(_consumerOptions.RejectPatientsIn.HasValue) - _fulfiller.Rejectors.Add(new PatientRejector(repositoryLocator.CatalogueRepository.GetObjectByID(_consumerOptions.RejectPatientsIn.Value))); + _fulfiller.Rejectors.Add(new ColumnInfoValuesRejector(repositoryLocator.CatalogueRepository.GetObjectByID(_consumerOptions.RejectPatientsIn.Value))); if(_consumerOptions.Blacklists != null) foreach (int id in _consumerOptions.Blacklists) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs new file mode 100644 index 000000000..db208a4a8 --- /dev/null +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs @@ -0,0 +1,51 @@ +using FAnsi.Discovery; +using MapsDirectlyToDatabaseTable; +using NLog; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.QueryBuilding; +using ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Data.Common; + + +namespace Microservices.CohortExtractor.Execution.RequestFulfillers +{ + + public class ColumnInfoValuesRejector : ColumnValuesRejector + { + + public ColumnInfoValuesRejector(ColumnInfo columnInfo) : base(columnInfo.GetRuntimeName(),FetchTable(columnInfo)) + { + + } + + private static HashSet FetchTable(ColumnInfo columnInfo) + { + var logger = LogManager.GetCurrentClassLogger(); + HashSet toReturn = new HashSet(StringComparer.CurrentCultureIgnoreCase); + + var qb = new QueryBuilder(limitationSQL: null, hashingAlgorithm: null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), columnInfo)); + + string sql = qb.SQL; + logger.Info("Running PatientID fetch SQL:" + sql); + + DiscoveredTable server = columnInfo.TableInfo.Discover(DataAccessContext.DataExport); + + using (DbConnection con = server.Database.Server.GetConnection()) + { + con.Open(); + DbCommand cmd = server.GetCommand(sql, con); + DbDataReader reader = cmd.ExecuteReader(); + + while (reader.Read()) + toReturn.Add(reader[0].ToString()); + } + + logger.Info($"Found {toReturn.Count} patients in the reject list"); + + return toReturn; + } + } +} diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs new file mode 100644 index 000000000..07c8afed8 --- /dev/null +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; + + +namespace Microservices.CohortExtractor.Execution.RequestFulfillers +{ + public class ColumnValuesRejector : IRejector + { + private readonly HashSet _rejectPatients; + private readonly string _columnToCheck; + + public ColumnValuesRejector(string column, HashSet values) + { + _columnToCheck = column; + _rejectPatients = values; + } + + public bool Reject(DbDataReader row, out string reason) + { + string patientId; + + try + { + // The patient ID is null + if (row[_columnToCheck] == DBNull.Value) + { + reason = null; + return false; + } + + patientId = (string)row[_columnToCheck]; + } + catch (IndexOutOfRangeException ex) + { + throw new IndexOutOfRangeException($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_columnToCheck}", ex); + } + + if (_rejectPatients.Contains(patientId)) + { + reason = "Patient was in reject list"; + return true; + } + + reason = null; + return false; + } + + } +} diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs deleted file mode 100644 index efafa5781..000000000 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/PatientRejector.cs +++ /dev/null @@ -1,77 +0,0 @@ -using FAnsi.Discovery; -using MapsDirectlyToDatabaseTable; -using NLog; -using Rdmp.Core.Curation.Data; -using Rdmp.Core.QueryBuilding; -using ReusableLibraryCode.DataAccess; -using System; -using System.Collections.Generic; -using System.Data.Common; - - -namespace Microservices.CohortExtractor.Execution.RequestFulfillers -{ - public class PatientRejector : IRejector - { - private readonly HashSet _rejectPatients = new HashSet(StringComparer.CurrentCultureIgnoreCase); - private readonly Logger _logger; - private readonly string _patientIdColumnName; - - public PatientRejector(ColumnInfo patientIdColumn) - { - _logger = LogManager.GetCurrentClassLogger(); - _patientIdColumnName = patientIdColumn.GetRuntimeName(); - - var qb = new QueryBuilder(limitationSQL: null, hashingAlgorithm: null); - qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), patientIdColumn)); - - string sql = qb.SQL; - _logger.Info("Running PatientID fetch SQL:" + sql); - - DiscoveredTable server = patientIdColumn.TableInfo.Discover(DataAccessContext.DataExport); - - using (DbConnection con = server.Database.Server.GetConnection()) - { - con.Open(); - DbCommand cmd = server.GetCommand(sql, con); - DbDataReader reader = cmd.ExecuteReader(); - - while (reader.Read()) - _rejectPatients.Add(reader[0].ToString()); - } - - _logger.Info($"Found {_rejectPatients.Count} patients in the reject list"); - } - - - public bool Reject(DbDataReader row, out string reason) - { - string patientId; - - try - { - // The patient ID is null - if (row[_patientIdColumnName] == DBNull.Value) - { - reason = null; - return false; - } - - patientId = (string)row[_patientIdColumnName]; - } - catch (IndexOutOfRangeException ex) - { - throw new IndexOutOfRangeException($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_patientIdColumnName}", ex); - } - - if (_rejectPatients.Contains(patientId)) - { - reason = "Patient was in reject list"; - return true; - } - - reason = null; - return false; - } - } -} diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs similarity index 64% rename from tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs rename to tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs index 6fc251277..550f60559 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/PatientRejectorTests.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs @@ -5,20 +5,20 @@ using NUnit.Framework; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; -using System; using System.Collections.Generic; using System.Data.Common; using Tests.Common; namespace Microservices.CohortExtractor.Tests { - public class PatientRejectorTests : DatabaseTests + + public class ColumnInfoValuesRejectorTests : DatabaseTests { private const string PatColName = "PatientID"; [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] - public void Test_PatientRejector(DatabaseType type) + public void Test_ColumnInfoValuesRejectorTests(DatabaseType type) { DiscoveredDatabase server = GetCleanedServer(type); DiscoveredTable tbl = server.CreateTable("BadPatients", new[] { new DatabaseColumnRequest(PatColName, "varchar(100)") }); @@ -30,7 +30,7 @@ public void Test_PatientRejector(DatabaseType type) new TableInfoImporter(CatalogueRepository, tbl).DoImport(out TableInfo _, out ColumnInfo[] cols); - var rejector = new PatientRejector(cols[0]); + var rejector = new ColumnInfoValuesRejector(cols[0]); var moqDave = new Mock(); moqDave.Setup(x => x[PatColName]) @@ -54,24 +54,5 @@ public void Test_PatientRejector(DatabaseType type) Assert.AreEqual("Patient was in reject list", reason); } - [TestCase(DatabaseType.MicrosoftSQLServer)] - [TestCase(DatabaseType.MySql)] - public void Test_PatientRejector_MissingColumn_Throws(DatabaseType type) - { - DiscoveredDatabase server = GetCleanedServer(type); - DiscoveredTable tbl = server.CreateTable("BadPatients", new[] { new DatabaseColumnRequest(PatColName, "varchar(100)") }); - - new TableInfoImporter(CatalogueRepository, tbl).DoImport(out TableInfo _, out ColumnInfo[] cols); - - var rejector = new PatientRejector(cols[0]); - - var moqDave = new Mock(); - moqDave - .Setup(x => x[PatColName]) - .Throws(); - - var exc = Assert.Throws(() => rejector.Reject(moqDave.Object, out string _)); - Assert.True(exc.Message.Contains($"Expected a column called {PatColName}")); - } } } diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs new file mode 100644 index 000000000..2e317e716 --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs @@ -0,0 +1,55 @@ +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Moq; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Data.Common; + +namespace Microservices.CohortExtractor.Tests +{ + public class ColumnValuesRejectorTests + { + private const string PatColName = "PatientID"; + + [Test] + public void Test_ColumnValuesRejector_MissingColumn_Throws() + { + var rejector = new ColumnValuesRejector("fff",new HashSet{ "dave","frank"}); + + var moqDave = new Mock(); + moqDave + .Setup(x => x["fff"]) + .Throws(); + + var exc = Assert.Throws(() => rejector.Reject(moqDave.Object, out string _)); + Assert.True(exc.Message.Contains($"Expected a column called fff")); + } + + [Test] + public void Test_ColumnValuesRejectorTests() + { + var rejector = new ColumnValuesRejector(PatColName,new HashSet(new []{ "Frank","Peter","David"},StringComparer.CurrentCultureIgnoreCase)); + + var moqDave = new Mock(); + moqDave.Setup(x => x[PatColName]) + .Returns("Dave"); + + Assert.IsFalse(rejector.Reject(moqDave.Object, out string reason)); + Assert.IsNull(reason); + + var moqFrank = new Mock(); + moqFrank.Setup(x => x[PatColName]) + .Returns("Frank"); + + Assert.IsTrue(rejector.Reject(moqFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + + var moqLowerCaseFrank = new Mock(); + moqLowerCaseFrank.Setup(x => x[PatColName]) + .Returns("frank"); + + Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + } + } +} From 4d348e654803aba173c3e33af7eb4ff4b7a42869 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 10 Sep 2020 17:51:12 +0100 Subject: [PATCH 122/138] add password entries to default config --- data/microserviceConfigs/default.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index d7f004e3e..aa5000b29 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -24,16 +24,19 @@ MongoDatabases: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'dicom' ExtractionStoreOptions: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'extraction' DeadLetterStoreOptions: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'deadLetterStorage' DeadLetterReprocessorOptions: From fd42ce74d153d355c90054e53d7d20b0cd44022a Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 10 Sep 2020 17:53:44 +0100 Subject: [PATCH 123/138] Allow decorators to be injected to GlobalOptionsFactory --- .../Options/EnvironmentVariableDecorator.cs | 19 ++++++-------- ...ionsFactory.cs => GlobalOptionsFactory.cs} | 26 +++++++++++++------ .../Smi.Common/Options/OptionsDecorator.cs | 12 ++++----- tests/common/Smi.Common.Tests/OptionsTests.cs | 13 +++++----- 4 files changed, 39 insertions(+), 31 deletions(-) rename src/common/Smi.Common/Options/{OptionsFactory.cs => GlobalOptionsFactory.cs} (72%) diff --git a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs index 633862d7b..c72276d0f 100644 --- a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs +++ b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs @@ -1,10 +1,7 @@ using System; -using System.Collections.Generic; -using System.Text; namespace Smi.Common.Options { - /// /// Populates values in based on environment variables /// @@ -12,23 +9,23 @@ public class EnvironmentVariableDecorator : OptionsDecorator { public override GlobalOptions Decorate(GlobalOptions options) { - ForAll(options,SetMongoPassword); - + ForAll(options, SetMongoPassword); + var logsRoot = Environment.GetEnvironmentVariable("SMI_LOGS_ROOT"); - - if(!string.IsNullOrWhiteSpace(logsRoot)) + + if (!string.IsNullOrWhiteSpace(logsRoot)) options.LogsRoot = logsRoot; return options; } - private MongoDbOptions SetMongoPassword(MongoDbOptions opt) + private static MongoDbOptions SetMongoPassword(MongoDbOptions opt) { //get the environment variables current value - var envVar = Environment.GetEnvironmentVariable("MONGO_SERVICE_PASSWORD"); + string envVar = Environment.GetEnvironmentVariable("MONGO_SERVICE_PASSWORD"); - //if theres an env var for it and there are mongodb options being used - if(!string.IsNullOrWhiteSpace(envVar) && opt != null) + //if there's an env var for it and there are mongodb options being used + if (!string.IsNullOrWhiteSpace(envVar) && opt != null) opt.Password = envVar; return opt; diff --git a/src/common/Smi.Common/Options/OptionsFactory.cs b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs similarity index 72% rename from src/common/Smi.Common/Options/OptionsFactory.cs rename to src/common/Smi.Common/Options/GlobalOptionsFactory.cs index 4bb0e3e56..7a6269876 100644 --- a/src/common/Smi.Common/Options/OptionsFactory.cs +++ b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs @@ -1,19 +1,29 @@ -using System; +using JetBrains.Annotations; +using System; using System.Collections.Generic; using System.IO; -using System.Text; using YamlDotNet.Serialization; namespace Smi.Common.Options { public class GlobalOptionsFactory { - public List Decorators {get;set;} = new List(); + private readonly List _decorators = new List(); - public GlobalOptionsFactory() + /// + /// Create a GlobalOptionsFactory with the given set of s. Adds a single by default if passed a null value. + /// + /// + public GlobalOptionsFactory( + [CanBeNull] ICollection decorators = null + ) { - Decorators.Add(new EnvironmentVariableDecorator()); + if (decorators != null) + _decorators.AddRange(decorators); + else + _decorators.Add(new EnvironmentVariableDecorator()); } + public GlobalOptions Load(string environment = "default", string currentDirectory = null) { IDeserializer deserializer = new DeserializerBuilder() @@ -41,13 +51,13 @@ public GlobalOptions Load(string environment = "default", string currentDirector } /// - /// Applies all to + /// Applies all to /// /// /// private GlobalOptions Decorate(GlobalOptions globals) { - foreach(var d in Decorators) + foreach (var d in _decorators) globals = d.Decorate(globals); return globals; @@ -56,7 +66,7 @@ private GlobalOptions Decorate(GlobalOptions globals) public GlobalOptions Load(CliOptions cliOptions) { //load but do not decorate - GlobalOptions globalOptions = Load(cliOptions.YamlFile,null); + GlobalOptions globalOptions = Load(cliOptions.YamlFile, null); globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); return globalOptions; diff --git a/src/common/Smi.Common/Options/OptionsDecorator.cs b/src/common/Smi.Common/Options/OptionsDecorator.cs index 84060e098..de524bcf3 100644 --- a/src/common/Smi.Common/Options/OptionsDecorator.cs +++ b/src/common/Smi.Common/Options/OptionsDecorator.cs @@ -7,27 +7,27 @@ public abstract class OptionsDecorator : IOptionsDecorator { public abstract GlobalOptions Decorate(GlobalOptions options); - protected void ForAll(IOptions globals,Func setter) where T: IOptions + protected static void ForAll(IOptions globals, Func setter) where T : IOptions { //for each property on branch - foreach(PropertyInfo p in globals.GetType().GetProperties()) + foreach (PropertyInfo p in globals.GetType().GetProperties()) { var currentValue = p.GetValue(globals); //if it's a T then call the action (note that we check the property Type because we are interested in the property even if it is null - if(p.PropertyType.IsAssignableFrom(typeof(T))) + if (p.PropertyType.IsAssignableFrom(typeof(T))) { //the delegate changes the value of the property of Type T (or creates a new instance from scratch) var result = setter((T)currentValue); //store the result of the delegate for this property - p.SetValue(globals,result); + p.SetValue(globals, result); } //process it's children - if(currentValue is IOptions subOptions) + if (currentValue is IOptions subOptions) { - ForAll(subOptions,setter); + ForAll(subOptions, setter); } } } diff --git a/tests/common/Smi.Common.Tests/OptionsTests.cs b/tests/common/Smi.Common.Tests/OptionsTests.cs index c98246db6..a3e6de994 100644 --- a/tests/common/Smi.Common.Tests/OptionsTests.cs +++ b/tests/common/Smi.Common.Tests/OptionsTests.cs @@ -1,7 +1,8 @@  -using System; using NUnit.Framework; using Smi.Common.Options; +using System; +using System.Collections.Generic; namespace Smi.Common.Tests { @@ -67,7 +68,7 @@ private class TestDecorator : OptionsDecorator { public override GlobalOptions Decorate(GlobalOptions options) { - ForAll(options,(o)=> new MongoDbOptions{DatabaseName = "FFFFF" }); + ForAll(options, (o) => new MongoDbOptions { DatabaseName = "FFFFF" }); return options; } } @@ -75,11 +76,11 @@ public override GlobalOptions Decorate(GlobalOptions options) [Test] public void TestDecorators() { - var factory = new GlobalOptionsFactory(){ Decorators = { new TestDecorator()} }; + var factory = new GlobalOptionsFactory(new List { new TestDecorator() }); var g = factory.Load(); - Assert.AreEqual("FFFFF",g.MongoDatabases.DeadLetterStoreOptions.DatabaseName); - Assert.AreEqual("FFFFF",g.MongoDatabases.DicomStoreOptions.DatabaseName); - Assert.AreEqual("FFFFF",g.MongoDatabases.ExtractionStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF", g.MongoDatabases.DeadLetterStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF", g.MongoDatabases.DicomStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF", g.MongoDatabases.ExtractionStoreOptions.DatabaseName); } } } From 52fe40ade8af1c225a2334d522a342b65d681965 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Thu, 10 Sep 2020 18:05:04 +0100 Subject: [PATCH 124/138] Clarify comment --- src/common/Smi.Common/Options/GlobalOptionsFactory.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/common/Smi.Common/Options/GlobalOptionsFactory.cs b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs index 7a6269876..e7278243b 100644 --- a/src/common/Smi.Common/Options/GlobalOptionsFactory.cs +++ b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs @@ -65,14 +65,13 @@ private GlobalOptions Decorate(GlobalOptions globals) public GlobalOptions Load(CliOptions cliOptions) { - //load but do not decorate GlobalOptions globalOptions = Load(cliOptions.YamlFile, null); globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); + // The above Load call does the decoration - don't do it here. return globalOptions; } - private object GetGlobalOption(Type arg) { return arg == typeof(GlobalOptions) ? From 21c68dd8552caad95656115001e3182e384c4265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2020 05:37:20 +0000 Subject: [PATCH 125/138] Bump MongoDB.Driver from 2.11.1 to 2.11.2 Bumps MongoDB.Driver from 2.11.1 to 2.11.2. Signed-off-by: dependabot[bot] --- src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj index b86c5c947..3e0ed68b4 100644 --- a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj +++ b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj @@ -14,7 +14,7 @@ - + From ac8253f2cb2d6f1f3b2408b9533124467ab1e272 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 11 Sep 2020 08:33:30 +0100 Subject: [PATCH 126/138] changed `RejectPatientsIn` to `RejectColumnInfos` and added support for referencing multiple columns --- src/common/Smi.Common/Options/GlobalOptions.cs | 4 ++-- .../Execution/CohortExtractorHost.cs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 1c93e1f59..64c3765fb 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -405,9 +405,9 @@ public string AuditorType public ProducerOptions ExtractFilesInfoProducerOptions { get; set; } /// - /// ID of a ColumnInfo that contains a list of patients who should not have data extracted for them. e.g. opt out + /// ID(s) of ColumnInfo that contains a list of values which should not have data extracted for them. e.g. opt out. The name of the column referenced must match a column in the extraction table /// - public int? RejectPatientsIn { get; set; } + public List RejectColumnInfos { get; set; } public override string ToString() { diff --git a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs index 781aa26c1..ba9d04375 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs @@ -132,8 +132,9 @@ private void InitializeExtractionSources(IRDMPPlatformRepositoryServiceLocator r if(!string.IsNullOrWhiteSpace(_consumerOptions.RejectorType)) _fulfiller.Rejectors.Add(ObjectFactory.CreateInstance(_consumerOptions.RejectorType,typeof(IRejector).Assembly)); - if(_consumerOptions.RejectPatientsIn.HasValue) - _fulfiller.Rejectors.Add(new ColumnInfoValuesRejector(repositoryLocator.CatalogueRepository.GetObjectByID(_consumerOptions.RejectPatientsIn.Value))); + if(_consumerOptions.RejectColumnInfos != null) + foreach(var id in _consumerOptions.RejectColumnInfos) + _fulfiller.Rejectors.Add(new ColumnInfoValuesRejector(repositoryLocator.CatalogueRepository.GetObjectByID(id))); if(_consumerOptions.Blacklists != null) foreach (int id in _consumerOptions.Blacklists) From 60253f09658eead21225fabd621a25109d7d9198 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 11 Sep 2020 08:37:29 +0100 Subject: [PATCH 127/138] Updated yaml comments --- data/microserviceConfigs/default.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index a301d277b..6c6ae96a8 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -62,17 +62,14 @@ CohortExtractorOptions: QoSPrefetchCount: 10000 AutoAck: false AllCatalogues: true - OnlyCatalogues: [1,2,3] - - # Set this to the ID of a ColumnInfo that contains patient IDs you want forbidden - #RejectPatientsIn: 1234 # List of IDs of Catalogues to extract from (in ascending order). # Ignored if "AllCatalogues == true" - # - 2 - # - 4 - # - 5 - # also doable on a single line with [2,4,5] :) + OnlyCatalogues: [1,2,3] + + # ID(s) of ColumnInfo that contains a list of values which should not have data extracted for them. e.g. opt out. The name of the column referenced must match a column in the extraction table + #RejectColumnInfos: [105,110] + AuditorType: 'Microservices.CohortExtractor.Audit.NullAuditExtractions' RequestFulfillerType: 'Microservices.CohortExtractor.Execution.RequestFulfillers.FromCataloguesExtractionRequestFulfiller' ProjectPathResolverType: 'Microservices.CohortExtractor.Execution.ProjectPathResolvers.DefaultProjectPathResolver' From fbad604a8944f25da75089b11172d5d385178498 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 11 Sep 2020 08:40:32 +0100 Subject: [PATCH 128/138] updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0342ec71b..c0a5aa763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) - IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` - Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool -- Added new rejector that throws out patients whose IDs are stored in a database table. Set `RejectPatientsIn` option in yaml to enable this +- Added new rejector that throws out values (e.g. patient IDs) whose IDs are stored in a database table. Set `RejectColumnInfos` option in yaml to enable this ## [1.11.1] - 2020-08-12 From dc72af02f11905a4c2d17088476774d334f77345 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 11 Sep 2020 13:18:43 +0100 Subject: [PATCH 129/138] Removed uneeded static keyword --- src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs | 2 +- src/common/Smi.Common/Options/OptionsDecorator.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs index c72276d0f..1b74ccf54 100644 --- a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs +++ b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs @@ -19,7 +19,7 @@ public override GlobalOptions Decorate(GlobalOptions options) return options; } - private static MongoDbOptions SetMongoPassword(MongoDbOptions opt) + private MongoDbOptions SetMongoPassword(MongoDbOptions opt) { //get the environment variables current value string envVar = Environment.GetEnvironmentVariable("MONGO_SERVICE_PASSWORD"); diff --git a/src/common/Smi.Common/Options/OptionsDecorator.cs b/src/common/Smi.Common/Options/OptionsDecorator.cs index de524bcf3..44b00a5c2 100644 --- a/src/common/Smi.Common/Options/OptionsDecorator.cs +++ b/src/common/Smi.Common/Options/OptionsDecorator.cs @@ -7,7 +7,7 @@ public abstract class OptionsDecorator : IOptionsDecorator { public abstract GlobalOptions Decorate(GlobalOptions options); - protected static void ForAll(IOptions globals, Func setter) where T : IOptions + protected void ForAll(IOptions globals, Func setter) where T : IOptions { //for each property on branch foreach (PropertyInfo p in globals.GetType().GetProperties()) From 013104647bf0c42914d260c49eb9eeed05d24254 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 06:10:31 +0000 Subject: [PATCH 130/138] Bump System.IO.Abstractions.TestingHelpers from 12.1.1 to 12.1.9 Bumps [System.IO.Abstractions.TestingHelpers](https://github.com/System-IO-Abstractions/System.IO.Abstractions) from 12.1.1 to 12.1.9. - [Release notes](https://github.com/System-IO-Abstractions/System.IO.Abstractions/releases) - [Commits](https://github.com/System-IO-Abstractions/System.IO.Abstractions/compare/v12.1.1...v12.1.9) Signed-off-by: dependabot[bot] --- .../Applications.DicomDirectoryProcessor.Tests.csproj | 2 +- .../Microservices.CohortPackager.Tests.csproj | 2 +- .../Microservices.DicomTagReader.Tests.csproj | 2 +- .../Microservices.FileCopier.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj index 99408ef2f..978359d8f 100644 --- a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj @@ -18,7 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj index 23a4e0a5b..fb59450aa 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj +++ b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj @@ -19,7 +19,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj index 22ab4b977..2dc6c0e52 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj +++ b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj @@ -18,7 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj index bae8325a6..3ea4d2464 100644 --- a/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj +++ b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj @@ -18,7 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From d0edba012d530a65ce0f07cd3a6ec91edb4c2d13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 06:10:35 +0000 Subject: [PATCH 131/138] Bump System.IO.Abstractions from 12.1.1 to 12.1.9 Bumps [System.IO.Abstractions](https://github.com/System-IO-Abstractions/System.IO.Abstractions) from 12.1.1 to 12.1.9. - [Release notes](https://github.com/System-IO-Abstractions/System.IO.Abstractions/releases) - [Commits](https://github.com/System-IO-Abstractions/System.IO.Abstractions/compare/v12.1.1...v12.1.9) Signed-off-by: dependabot[bot] --- src/common/Smi.Common/Smi.Common.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 9ddf36651..be5c1ee46 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -44,7 +44,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 5b8a64d742718abf980f23445184a7dd47f356b7 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 10:51:05 +0100 Subject: [PATCH 132/138] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2973471e..780475e98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` - Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool - Added new rejector that throws out values (e.g. patient IDs) whose IDs are stored in a database table. Set `RejectColumnInfos` option in yaml to enable this - +- Added a check to QueryToExecuteResult for RejectReason being null when Reject is true. ### Changed - Environment variables are no longer required. Previous settings now appear in configuration file From e9dcb5ccd62237c9a776af981015296f06b3e072 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 10:57:58 +0100 Subject: [PATCH 133/138] Add test for QueryToExecuteResult --- .../QueryToExecuteResultTest.cs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs new file mode 100644 index 000000000..f5b38b7cf --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs @@ -0,0 +1,54 @@ +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using NUnit.Framework; +using Smi.Common.Tests; +using System; + + +namespace Microservices.CohortExtractor.Tests.Execution.RequestFulfillers +{ + public class QueryToExecuteResultTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + /// + /// Asserts that we always have a rejection reason when rejection=true + /// + [Test] + public void Test_QueryToExecuteResult_RejectReasonNullOrEmpty_ThrowsException() + { + Assert.Throws(() => + { + var _ = new QueryToExecuteResult("foo", "bar", "baz", "whee", rejection: true, rejectionReason: null); + }); + Assert.Throws(() => + { + var _ = new QueryToExecuteResult("foo", "bar", "baz", "whee", rejection: true, rejectionReason: " "); + }); + } + + #endregion + } +} From fb3c864eefcb0726d2d1fc7bc56f00f2d56c236e Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 12:31:30 +0000 Subject: [PATCH 134/138] add releasing notes --- docs/RELEASING.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/RELEASING.md diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 000000000..eb4b6e4ea --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,49 @@ +# SmiServices Release Process + +The steps to cut a new release of SmiServices are as follows + +## Creating A Normal Release + +- First, identify the next release version. This can be determined by looking at the last release and deciding if the new code to be released is a major, minor, or patch change as per [semver](https://semver.org). E.g. if the previous release was `v1.2.3` and only new non-breaking features are in the `Unreleased` section of the CHANGELOG, then the next release should be`v1.3.0`. The definition of "breaking" can often be subjective though, so other members of the team if you're unsure. + +- Ensure you are on the latest commit on the `develop` branch , and create a new release branch: + + ```console + $ git st + On branch develop + Your branch is up to date with 'origin/develop'. + + nothing added to commit but untracked files present (use "git add" to track) + + $ git pull + Already up to date. + + $ git checkout -b release/v1.3.0 + Switched to a new branch 'release/v1.3.0' + ``` + +- Update any files referencing the version. Currently these are: + - `CHANGELOG.md`: Add a new tag under the `Unreleased` section, and add a new link to the bottom. Look at a previous release for an example e.g. `git show 827a1a`. + - `README.md`: Bump the version + - `src/SharedAssemblyInfo.cs`: Bump the versions + +- Commit these changes and push the new branch with the message "Start release branch for v1.2.3" +- Open a PR for this branch with the title "Release ". Request a review from `@tznind` and `@rkm`. +- If there are any further changes which need to be included in the release, then these can be merged into the release branch from `develop`. +- Wait for the PR to be merged +- Checkout `master` and pull the merge commit +- Tag the release, e.g.: + + ```console + $ git tag v1.2.3 + $ git push origin v1.2.3 + ``` + +- Merge `master` back into `develop` to ensure that any changes from the release branch are present +- Delete the release branch +- Wait for Travis to build the tagged commit +- Check that the built binaries are added to the [releases](https://github.com/SMI/SmiServices/releases) page. Update the title and description using the CHANGELOG. + +## Creating A Hotfix Release + +TODO From a43671e4c100ec7659a3f53ac580a1b0ed0ca1ac Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 12:33:17 +0000 Subject: [PATCH 135/138] Start release branch for v1.12.0 --- CHANGELOG.md | 7 +++++-- README.md | 2 +- src/SharedAssemblyInfo.cs | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 780475e98..d34fff059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +## [1.12.0] - 2020-09-14 + - Add SecurityCodeScan tool to build chain for .Net code - Extraction report: Group PixelData separately and sort by length - Fix the extraction output directory to be `/extractions/` @@ -21,7 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added a check to QueryToExecuteResult for RejectReason being null when Reject is true. ### Changed -- Environment variables are no longer required. Previous settings now appear in configuration file +- [breaking] Environment variables are no longer required. Previous settings now appear in configuration file - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` - Removed `ISIDENTIFIABLE_NUMTHREADS` as it didn't work correctly anyway @@ -386,7 +388,8 @@ First stable release after importing the repository from the private [SMIPlugin] - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported. -[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.11.1...develop +[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.12.0...develop +[1.12.0]: https://github.com/SMI/SmiServices/compare/v1.11.1...v1.12.0 [1.11.1]: https://github.com/SMI/SmiServices/compare/v1.11.0...v1.11.1 [1.11.0]: https://github.com/SMI/SmiServices/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/SMI/SmiServices/compare/v1.9.0...v1.10.0 diff --git a/README.md b/README.md index ca33497bd..9cb01a16c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/) [![Coverage Status](https://coveralls.io/repos/github/SMI/SmiServices/badge.svg)](https://coveralls.io/github/SMI/SmiServices) -Version: `1.11.1` +Version: `1.12.0` # SMI Services diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index abfe624aa..9520fe819 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -7,6 +7,6 @@ [assembly: AssemblyCulture("")] // These should be overwritten by release builds -[assembly: AssemblyVersion("1.11.1")] -[assembly: AssemblyFileVersion("1.11.1")] -[assembly: AssemblyInformationalVersion("1.11.1")] // This one can have the extra build info after it +[assembly: AssemblyVersion("1.12.0")] +[assembly: AssemblyFileVersion("1.12.0")] +[assembly: AssemblyInformationalVersion("1.12.0")] // This one can have the extra build info after it From b5bf725650eb1ad15ef3ed9bbf69304c04619b08 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 12:39:54 +0000 Subject: [PATCH 136/138] update RELEASING --- docs/RELEASING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index eb4b6e4ea..e294d9561 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -22,15 +22,15 @@ The steps to cut a new release of SmiServices are as follows Switched to a new branch 'release/v1.3.0' ``` -- Update any files referencing the version. Currently these are: - - `CHANGELOG.md`: Add a new tag under the `Unreleased` section, and add a new link to the bottom. Look at a previous release for an example e.g. `git show 827a1a`. +- Update any files referencing the version. To see an example, check the previous release commit: `git log --all --grep="Start release branch" -1 --name-only --format=`. E.g.: + - `CHANGELOG.md`: Add a new section header under `Unreleased`, and add a new link to the bottom - `README.md`: Bump the version - `src/SharedAssemblyInfo.cs`: Bump the versions - Commit these changes and push the new branch with the message "Start release branch for v1.2.3" - Open a PR for this branch with the title "Release ". Request a review from `@tznind` and `@rkm`. - If there are any further changes which need to be included in the release, then these can be merged into the release branch from `develop`. -- Wait for the PR to be merged +- Wait for the PR to be reviewed and merged - Checkout `master` and pull the merge commit - Tag the release, e.g.: From 0ad1649b84134b4be625a8d2f15ae96f399ed946 Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 12:47:34 +0000 Subject: [PATCH 137/138] sort changelog --- CHANGELOG.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d34fff059..724768bd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,25 +8,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [1.12.0] - 2020-09-14 -- Add SecurityCodeScan tool to build chain for .Net code -- Extraction report: Group PixelData separately and sort by length -- Fix the extraction output directory to be `/extractions/` -- Add identifiable extraction support +### Added + +- [breaking] Add identifiable extraction support - New service "FileCopier" which sits in place of CTP for identifiable extractions and copies source files to their output dirs - Changes to MongoDB extraction schema, but backwards compatibility has been tested - - [breaking] RabbitMQ extraction config has been refactored. Queues and service config files need to be updated + - RabbitMQ extraction config has been refactored. Queues and service config files need to be updated +- Add [SecurityCodeScan](https://security-code-scan.github.io/) tool to build chain for .NET code - Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions -- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) -- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` - Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool - Added new rejector that throws out values (e.g. patient IDs) whose IDs are stored in a database table. Set `RejectColumnInfos` option in yaml to enable this - Added a check to QueryToExecuteResult for RejectReason being null when Reject is true. ### Changed + - [breaking] Environment variables are no longer required. Previous settings now appear in configuration file - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` - Removed `ISIDENTIFIABLE_NUMTHREADS` as it didn't work correctly anyway +- Extraction report: Group PixelData separately and sort by length +- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) +- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` + +### Fixed + +- Fix the extraction output directory to be `/extractions/` ## [1.11.1] - 2020-08-12 From ee4765f7eb5cbe9286503cfb2ae307c6699f108b Mon Sep 17 00:00:00 2001 From: Ruairidh MacLeod Date: Mon, 14 Sep 2020 14:18:25 +0000 Subject: [PATCH 138/138] add dependencies to changelog --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 724768bd3..46730e8d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix the extraction output directory to be `/extractions/` +### Dependencies + +- Bump fo-dicom.Drawing from 4.0.5 to 4.0.6 +- Bump fo-dicom.NetCore from 4.0.5 to 4.0.6 +- Bump HIC.BadMedicine.Dicom from 0.0.6 to 0.0.7 +- Bump HIC.DicomTypeTranslation from 2.3.0 to 2.3.1 +- Bump HIC.FAnsiSql from 1.0.2 to 1.0.5 +- Bump HIC.RDMP.Dicom from 2.1.6 to 2.1.10 +- Bump HIC.RDMP.Plugin from 4.1.6 to 4.1.8 +- Bump HIC.RDMP.Plugin.Test from 4.1.6 to 4.1.8 +- Bump Microsoft.CodeAnalysis.CSharp.Scripting from 3.6.0 to 3.7.0 +- Bump Microsoft.Extensions.Caching.Memory from 3.1.6 to 3.1.8 +- Bump Microsoft.NET.Test.Sdk from 16.6.1 to 16.7.1 +- Bump MongoDB.Driver from 2.11.0 to 2.11.2 +- Bump System.IO.Abstractions from 12.1.1 to 12.1.9 +- Bump System.IO.Abstractions.TestingHelpers from 12.1.1 to 12.1.9 +- Bump Terminal.Gui from 0.81.0 to 0.89.4 + ## [1.11.1] - 2020-08-12 - Set PublishTrimmed to false to fix bug with missing assemblies in prod.