diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f698018d..46730e8d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,52 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +## [1.12.0] - 2020-09-14 + +### Added + +- [breaking] Add identifiable extraction support + - New service "FileCopier" which sits in place of CTP for identifiable extractions and copies source files to their output dirs + - Changes to MongoDB extraction schema, but backwards compatibility has been tested + - RabbitMQ extraction config has been refactored. Queues and service config files need to be updated +- Add [SecurityCodeScan](https://security-code-scan.github.io/) tool to build chain for .NET code +- Add "no filters" extraction support. If specified when running ExtractorCLI, no file rejection filters will be applied by CohortExtractor. True by default for identifiable extractions +- Added caching of values looked up in NLP/rulesbase for IsIdentifiable tool +- Added new rejector that throws out values (e.g. patient IDs) whose IDs are stored in a database table. Set `RejectColumnInfos` option in yaml to enable this +- Added a check to QueryToExecuteResult for RejectReason being null when Reject is true. + +### Changed + +- [breaking] Environment variables are no longer required. Previous settings now appear in configuration file + - Environment variable `SMI_LOGS_ROOT` is now `GlobalOptions.LogsRoot` + - Environment variable `MONGO_SERVICE_PASSWORD` is now `MongoDbOptions.Password` + - Removed `ISIDENTIFIABLE_NUMTHREADS` as it didn't work correctly anyway +- Extraction report: Group PixelData separately and sort by length +- IsIdentifiable Reviewer 'Symbols' rule factory now supports digits only or characters only mode (e.g. use `\d` for digits but leave characters verbatim) +- IsIdentifiable Reviewer 'symbols' option when building Regex now builds capture groups and matches only the failing parts of the input string not the full ProblemValue. For example `MR Head 12-11-20` would return `(\d\d-\d\d-\d\d)$` + +### Fixed + +- Fix the extraction output directory to be `/extractions/` + +### Dependencies + +- Bump fo-dicom.Drawing from 4.0.5 to 4.0.6 +- Bump fo-dicom.NetCore from 4.0.5 to 4.0.6 +- Bump HIC.BadMedicine.Dicom from 0.0.6 to 0.0.7 +- Bump HIC.DicomTypeTranslation from 2.3.0 to 2.3.1 +- Bump HIC.FAnsiSql from 1.0.2 to 1.0.5 +- Bump HIC.RDMP.Dicom from 2.1.6 to 2.1.10 +- Bump HIC.RDMP.Plugin from 4.1.6 to 4.1.8 +- Bump HIC.RDMP.Plugin.Test from 4.1.6 to 4.1.8 +- Bump Microsoft.CodeAnalysis.CSharp.Scripting from 3.6.0 to 3.7.0 +- Bump Microsoft.Extensions.Caching.Memory from 3.1.6 to 3.1.8 +- Bump Microsoft.NET.Test.Sdk from 16.6.1 to 16.7.1 +- Bump MongoDB.Driver from 2.11.0 to 2.11.2 +- Bump System.IO.Abstractions from 12.1.1 to 12.1.9 +- Bump System.IO.Abstractions.TestingHelpers from 12.1.1 to 12.1.9 +- Bump Terminal.Gui from 0.81.0 to 0.89.4 + ## [1.11.1] - 2020-08-12 - Set PublishTrimmed to false to fix bug with missing assemblies in prod. @@ -366,7 +412,8 @@ First stable release after importing the repository from the private [SMIPlugin] - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported. -[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.11.1...develop +[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.12.0...develop +[1.12.0]: https://github.com/SMI/SmiServices/compare/v1.11.1...v1.12.0 [1.11.1]: https://github.com/SMI/SmiServices/compare/v1.11.0...v1.11.1 [1.11.0]: https://github.com/SMI/SmiServices/compare/v1.10.0...v1.11.0 [1.10.0]: https://github.com/SMI/SmiServices/compare/v1.9.0...v1.10.0 diff --git a/PACKAGES.md b/PACKAGES.md index 16004cb8f..f80452f56 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -7,32 +7,33 @@ 2. This package is widely used and is actively maintained. 3. It is open source. -| Package | Source Code | Version | License | Purpose | Additional Risk Assessment | -| ------- | ------------| --------| ------- | ------- | -------------------------- | -| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.8.0](https://www.nuget.org/packages/CommandLineParser/2.8.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | -| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [15.0.5](https://www.nuget.org/packages/CsvHelper/15.0.5) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | -| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.5](https://www.nuget.org/packages/fo-dicom.NetCore/4.0.5) | [MS-PL](https://opensource.org/licenses/MS-PL) | | | -| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.3.0](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.3.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | -| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [1.0.2](https://www.nuget.org/packages/HIC.FAnsiSql/1.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | | -| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.1.6](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | -| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.1.6](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.1.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | -| JetBrains.Annotations | | [2020.1.0](https://www.nuget.org/packages/JetBrains.Annotations/2020.1.0) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | -| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [7.21.1](https://www.nuget.org/packages/Magick.NET-Q16-AnyCPU/7.21.1) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | | -| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [3.6.0](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp.Scripting/3.6.0) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | | -| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [3.1.6](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.6/License) | Caching ID mappings retrieved from Redis/MySQL | -| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.11.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.0/License) | For writting/reading dicom tags into MongoDb databases| -| NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | -| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | -| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.1.58](https://www.nuget.org/packages/StackExchange.Redis/2.1.58) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | | -| Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | | -| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Drawing.Common/4.7.0) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | | -| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [12.1.1](https://www.nuget.org/packages/System.IO.Abstractions/12.1.1) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | | -| System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | File I/O | | -| System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | -| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) | [0.81.0](https://www.nuget.org/packages/Terminal.Gui/0.81.0) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | | -| Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | -| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [8.1.2](https://www.nuget.org/packages/YamlDotNet/8.1.2) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.5](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.5) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | -| coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [1.4.2](https://www.nuget.org/packages/coveralls.io/1.4.2) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io | | -| OpenCover | [GitHub](https://github.com/OpenCover/opencover) | [4.7.922](https://www.nuget.org/packages/OpenCover/4.7.922) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE) | Calculates code coverage for tests| | +| Package | Source Code | License | Purpose +| ------- | ------------| ------- | ------- | +| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | +| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | +| fo-dicom.NetCore | [GitHub](https://github.com/fo-dicom/fo-dicom) | [MS-PL](https://opensource.org/licenses/MS-PL) | | +| HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | +| HIC.FAnsiSql | [GitHub](https://github.com/HicServices/FansiSql) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Database abstraction layer | +| HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | +| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | +| JetBrains.Annotations | |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | +| Magick.NET-Q16-AnyCPU | [GitHub](https://github.com/dlemstra/Magick.NET) | [Apache License v2](https://github.com/dlemstra/Magick.NET/blob/master/License.txt) | The .NET library for [ImageMagick](https://imagemagick.org/index.php) | +| Microsoft.CodeAnalysis.CSharp.Scripting | [GitHub](https://github.com/dotnet/roslyn) | [MIT](https://opensource.org/licenses/MIT) | Supports dynamic rules for cohort extraction logic | +| Microsoft.Extensions.Caching.Memory | [GitHub](https://github.com/dotnet/extensions) | [Apache 2.0](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.7/License) | Caching ID mappings retrieved from Redis/MySQL +| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) | [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.11.1/License) | For writting/reading dicom tags into MongoDb databases +| NLog | [GitHub](https://github.com/NLog/NLog) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | +| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission +| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | +| SecurityCodeScan | [GitHub](https://security-code-scan.github.io/) | [LGPL 3.0](https://opensource.org/licenses/lgpl-3.0.html) | Scans code for security issues during build | +| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | +| Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | +| System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | +| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | +| System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) |[MIT](https://opensource.org/licenses/MIT) | File I/O | +| System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| +| Terminal.Gui | [GitHub](https://github.com/migueldeicaza/gui.cs/) |[MIT](https://opensource.org/licenses/MIT) | Console GUI library | +| Tesseract | [GitHub](https://github.com/charlesw/tesseract/) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| +| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files +| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | +| coveralls.io | [GitHub](https://github.com/coveralls-net/coveralls.net) | [GNU](https://github.com/coveralls-net/coveralls.net#license)| Uploader for dot net coverage reports to Coveralls.io | +| OpenCover | [GitHub](https://github.com/OpenCover/opencover) |[MIT Compatible](https://github.com/OpenCover/opencover/blob/master/LICENSE) | Calculates code coverage for tests| diff --git a/README.md b/README.md index 82cdaad2b..9cb01a16c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/) [![Coverage Status](https://coveralls.io/repos/github/SMI/SmiServices/badge.svg)](https://coveralls.io/github/SMI/SmiServices) -Version: `1.11.1` +Version: `1.12.0` # SMI Services @@ -21,7 +21,6 @@ The latest binaries can be downloaded from the [releases section](https://github 1. [Microservices](#microservices) 1. [Data Load Microservices](#data-load-microservices) 2. [Image Extraction Microservices](#image-extraction-microservices) -1. [Environment Variables](#environment-variables) 2. [Solution Overivew](#solution-overview) 3. [Building](#building) 4. [Testing](#testing) @@ -74,19 +73,6 @@ A control queue is provided for controlling Microservices during runtime. It su | Fatal Error Logging | All Microservices that crash or log a fatal error are shut down and log a message to the Fatal Error Logging Exchange. TODO: Nobody listens to this currently.| | Quarantine | TODO: Doesn't exist yet.| -## Environment Variables - -The following environment variables are expected by the program: - ->TODO can we move `ISIDENTIFIABLE_NUMTHREADS` to config yaml/CLI? - -| Name | Purpose | Example | -|------|---------|---------| -| SMI_LOGS_ROOT | Required to be set if `ForceSmiLogsRoot` is set to `true` in the service config. Determines where log files are written to | `/var/log/smi` | -| MONGO_SERVICE_PASSWORD | Password that should be used to access the MongoDb database used by ETL pipeline services | MyPassword| -| ISIDENTIFIABLE_NUMTHREADS | Optional (defaults to 1). The number of threads to use when looking for identifiable data in extraction pipeline | 1| -| CI | When running tests in a CI, this ensures that integration tests are failed instead of skipped | 1| - ## Solution Overview Appart from the Microservices (documented above) the following library classes are also included in the solution: diff --git a/SmiServices.sln b/SmiServices.sln index a34da7b34..dbb817ecf 100644 --- a/SmiServices.sln +++ b/SmiServices.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.29411.108 @@ -75,6 +75,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.IsIdentifiabl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IsIdentifiableReviewer", "src\applications\IsIdentifiableReviewer\IsIdentifiableReviewer.csproj", "{C2031E86-81B4-405A-A923-9B82E0CE196F}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier", "src\microservices\Microservices.FileCopier\Microservices.FileCopier.csproj", "{D4E52707-FFF7-41E6-8057-C6DB344B8CD7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microservices.FileCopier.Tests", "tests\microservices\Microservices.FileCopier.Tests\Microservices.FileCopier.Tests.csproj", "{D61F6BF9-E857-457C-B745-40489A8CFE65}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -181,6 +185,14 @@ Global {C2031E86-81B4-405A-A923-9B82E0CE196F}.Debug|x64.Build.0 = Debug|x64 {C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.ActiveCfg = Release|x64 {C2031E86-81B4-405A-A923-9B82E0CE196F}.Release|x64.Build.0 = Release|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.ActiveCfg = Debug|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Debug|x64.Build.0 = Debug|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.ActiveCfg = Release|x64 + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7}.Release|x64.Build.0 = Release|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Debug|x64.ActiveCfg = Debug|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Debug|x64.Build.0 = Debug|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Release|x64.ActiveCfg = Release|x64 + {D61F6BF9-E857-457C-B745-40489A8CFE65}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -211,6 +223,8 @@ Global {1A27E9E8-F16E-43F9-927E-5FE92E2F97D8} = {421CCD37-3817-4748-B184-A134E19DD75C} {E632E673-0766-4A4D-ABE3-4B6D4F5BEFE2} = {421CCD37-3817-4748-B184-A134E19DD75C} {C2031E86-81B4-405A-A923-9B82E0CE196F} = {8B943F2C-835B-484A-86D2-3F1462970605} + {D4E52707-FFF7-41E6-8057-C6DB344B8CD7} = {421CCD37-3817-4748-B184-A134E19DD75C} + {D61F6BF9-E857-457C-B745-40489A8CFE65} = {421CCD37-3817-4748-B184-A134E19DD75C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {11CDEA53-71E8-4A9B-BC0D-74F4EB54F740} diff --git a/appveyor.yml b/appveyor.yml index 68b333e6c..c60232208 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,6 @@ version: 1.0.{build} init: + - cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - git config --global core.autocrlf true image: Visual Studio 2019 services: @@ -11,10 +12,9 @@ cache: - '%USERPROFILE%\.nuget\packages -> **\*.csproj' before_build: -- cmd: if defined APPVEYOR_PULL_REQUEST_NUMBER appveyor exit - ps: "Add-Content c:\\mongodb\\mongod.cfg \"`r`nreplication:`r`n replSetName: rs0`r`n\"" - cmd: "net start mongodb" -- cmd: "c:\\mongodb\\bin\\mongo --eval 'rs.initiate()'" +- cmd: "c:\\mongodb\\bin\\mongo --eval \"printjson(rs.initiate())\"" - choco install opencover.portable - choco install rabbitmq diff --git a/data/microserviceConfigs/default.yaml b/data/microserviceConfigs/default.yaml index 50bd36646..75ecb218e 100644 --- a/data/microserviceConfigs/default.yaml +++ b/data/microserviceConfigs/default.yaml @@ -24,16 +24,19 @@ MongoDatabases: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'dicom' ExtractionStoreOptions: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'extraction' DeadLetterStoreOptions: HostName: 'localhost' Port: 27017 UserName: '' + Password: '' DatabaseName: 'deadLetterStorage' DeadLetterReprocessorOptions: @@ -62,16 +65,19 @@ CohortExtractorOptions: QoSPrefetchCount: 10000 AutoAck: false AllCatalogues: true - OnlyCatalogues: [1,2,3] + # List of IDs of Catalogues to extract from (in ascending order). # Ignored if "AllCatalogues == true" - # - 2 - # - 4 - # - 5 - # also doable on a single line with [2,4,5] :) + OnlyCatalogues: [1,2,3] + + # ID(s) of ColumnInfo that contains a list of values which should not have data extracted for them. e.g. opt out. The name of the column referenced must match a column in the extraction table + #RejectColumnInfos: [105,110] + AuditorType: 'Microservices.CohortExtractor.Audit.NullAuditExtractions' RequestFulfillerType: 'Microservices.CohortExtractor.Execution.RequestFulfillers.FromCataloguesExtractionRequestFulfiller' ProjectPathResolverType: 'Microservices.CohortExtractor.Execution.ProjectPathResolvers.DefaultProjectPathResolver' + ExtractAnonRoutingKey: anon + ExtractIdentRoutingKey: ident # Writes (Producer) to this exchange ExtractFilesProducerOptions: ExchangeName: 'TEST.ExtractFileExchange' @@ -93,12 +99,12 @@ CohortPackagerOptions: QueueName: 'TEST.FileCollectionInfoQueue' QoSPrefetchCount: 1 AutoAck: false - AnonFailedOptions: - QueueName: 'TEST.AnonFailedQueue' + NoVerifyStatusOptions: + QueueName: 'TEST.ExtractedFileNoVerifyQueue' QoSPrefetchCount: 1 AutoAck: false VerificationStatusOptions: - QueueName: 'TEST.VerificationStatusQueue' + QueueName: 'TEST.ExtractedFileVerifiedQueue' QoSPrefetchCount: 1 AutoAck: false @@ -158,11 +164,22 @@ ProcessDirectoryOptions: MaxConfirmAttempts: 1 CTPAnonymiserOptions: - ExtractFileConsumerOptions: - QueueName: 'TEST.ExtractFileQueue' + VerifyRoutingKey: verify + NoVerifyRoutingKey: noverify + AnonFileConsumerOptions: + QueueName: 'TEST.ExtractFileAnonQueue' QoSPrefetchCount: 1 AutoAck: false ExtractFileStatusProducerOptions: + ExchangeName: 'TEST.ExtractedFileStatusExchange' + MaxConfirmAttempts: 1 + +FileCopierOptions: + NoVerifyRoutingKey: noverify + QueueName: 'TEST.ExtractFileIdentQueue' + QoSPrefetchCount: 1 + AutoAck: false + CopyStatusProducerOptions: ExchangeName: 'TEST.FileStatusExchange' MaxConfirmAttempts: 1 @@ -176,11 +193,11 @@ ExtractorClOptions: MaxConfirmAttempts: 1 IsIdentifiableOptions: - QueueName: 'TEST.IsIdentifiableQueue' + QueueName: 'TEST.ExtractedFileToVerifyQueue' QoSPrefetchCount: 1 AutoAck: false IsIdentifiableProducerOptions: - ExchangeName: 'TEST.IsIdentifiableExchange' + ExchangeName: 'TEST.ExtractedFileVerifiedExchange' MaxConfirmAttempts: 1 ClassifierType: 'Microservices.IsIdentifiable.Service.TesseractStanfordDicomFileClassifier' DataDirectory: '' diff --git a/data/rabbitmqConfigs/README.md b/data/rabbitmqConfigs/README.md index 445191edb..6155ae0bd 100644 --- a/data/rabbitmqConfigs/README.md +++ b/data/rabbitmqConfigs/README.md @@ -31,6 +31,15 @@ $ curl \ http://0.0.0.0:15672/api/definitions ``` +## Deleting a vhost + +```bash +> curl \ + -u guest:guest \ + -XDELETE \ + http://0.0.0.0:15672/api/vhosts/ +``` + ## Filter the default exchanges RabbitMQ has predefined exchanges which can't be removed from the management UI. To filter these out, tick the `regex` checkbox next to the search box on the `Exchanges` tab, then use this regex: diff --git a/data/rabbitmqConfigs/defaultExtractConfig.json b/data/rabbitmqConfigs/defaultExtractConfig.json index 80b2bbe9d..2453b7ed8 100644 --- a/data/rabbitmqConfigs/defaultExtractConfig.json +++ b/data/rabbitmqConfigs/defaultExtractConfig.json @@ -27,7 +27,7 @@ "arguments": {} }, { - "name": "AnonStatusExchange", + "name": "ExtractedFileStatusExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -36,7 +36,7 @@ "arguments": {} }, { - "name": "VerificationStatusExchange", + "name": "ExtractedFileVerifiedExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -54,7 +54,7 @@ "arguments": {} }, { - "name": "AnonFileExchange", + "name": "ExtractFileExchange", "vhost": "smi_extract", "type": "direct", "durable": true, @@ -115,21 +115,28 @@ "arguments": {} }, { - "name": "AnonSuccessQueue", + "name": "ExtractedFileToVerifyQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, "arguments": {} }, { - "name": "AnonFailedQueue", + "name": "ExtractedFileNoVerifyQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, "arguments": {} }, { - "name": "AnonFileQueue", + "name": "ExtractFileAnonQueue", + "vhost": "smi_extract", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "ExtractFileIdentQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, @@ -150,7 +157,7 @@ "arguments": {} }, { - "name": "VerificationStatusQueue", + "name": "ExtractedFileVerifiedQueue", "vhost": "smi_extract", "durable": true, "auto_delete": false, @@ -182,27 +189,35 @@ "arguments": {} }, { - "source": "AnonFileExchange", + "source": "ExtractFileExchange", "vhost": "smi_extract", - "destination": "AnonFileQueue", + "destination": "ExtractFileAnonQueue", "destination_type": "queue", - "routing_key": "", + "routing_key": "anon", + "arguments": {} + }, + { + "source": "ExtractFileExchange", + "vhost": "smi_extract", + "destination": "ExtractFileIdentQueue", + "destination_type": "queue", + "routing_key": "ident", "arguments": {} }, { - "source": "AnonStatusExchange", + "source": "ExtractedFileStatusExchange", "vhost": "smi_extract", - "destination": "AnonSuccessQueue", + "destination": "ExtractedFileToVerifyQueue", "destination_type": "queue", - "routing_key": "success", + "routing_key": "verify", "arguments": {} }, { - "source": "AnonStatusExchange", + "source": "ExtractedFileStatusExchange", "vhost": "smi_extract", - "destination": "AnonFailedQueue", + "destination": "ExtractedFileNoVerifyQueue", "destination_type": "queue", - "routing_key": "failure", + "routing_key": "noverify", "arguments": {} }, { @@ -222,9 +237,9 @@ "arguments": {} }, { - "source": "VerificationStatusExchange", + "source": "ExtractedFileVerifiedExchange", "vhost": "smi_extract", - "destination": "VerificationStatusQueue", + "destination": "ExtractedFileVerifiedQueue", "destination_type": "queue", "routing_key": "", "arguments": {} diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 000000000..e294d9561 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,49 @@ +# SmiServices Release Process + +The steps to cut a new release of SmiServices are as follows + +## Creating A Normal Release + +- First, identify the next release version. This can be determined by looking at the last release and deciding if the new code to be released is a major, minor, or patch change as per [semver](https://semver.org). E.g. if the previous release was `v1.2.3` and only new non-breaking features are in the `Unreleased` section of the CHANGELOG, then the next release should be`v1.3.0`. The definition of "breaking" can often be subjective though, so other members of the team if you're unsure. + +- Ensure you are on the latest commit on the `develop` branch , and create a new release branch: + + ```console + $ git st + On branch develop + Your branch is up to date with 'origin/develop'. + + nothing added to commit but untracked files present (use "git add" to track) + + $ git pull + Already up to date. + + $ git checkout -b release/v1.3.0 + Switched to a new branch 'release/v1.3.0' + ``` + +- Update any files referencing the version. To see an example, check the previous release commit: `git log --all --grep="Start release branch" -1 --name-only --format=`. E.g.: + - `CHANGELOG.md`: Add a new section header under `Unreleased`, and add a new link to the bottom + - `README.md`: Bump the version + - `src/SharedAssemblyInfo.cs`: Bump the versions + +- Commit these changes and push the new branch with the message "Start release branch for v1.2.3" +- Open a PR for this branch with the title "Release ". Request a review from `@tznind` and `@rkm`. +- If there are any further changes which need to be included in the release, then these can be merged into the release branch from `develop`. +- Wait for the PR to be reviewed and merged +- Checkout `master` and pull the merge commit +- Tag the release, e.g.: + + ```console + $ git tag v1.2.3 + $ git push origin v1.2.3 + ``` + +- Merge `master` back into `develop` to ensure that any changes from the release branch are present +- Delete the release branch +- Wait for Travis to build the tagged commit +- Check that the built binaries are added to the [releases](https://github.com/SMI/SmiServices/releases) page. Update the title and description using the CHANGELOG. + +## Creating A Hotfix Release + +TODO diff --git a/docs/extraction/Extraction-Pipeline-2.2.1.drawio b/docs/extraction/Extraction-Pipeline-2.2.1.drawio deleted file mode 100644 index 0e0121bb4..000000000 --- a/docs/extraction/Extraction-Pipeline-2.2.1.drawio +++ /dev/null @@ -1 +0,0 @@ -7V1tc5u4Fv41ntl7Z+IBxIv52CRNN7tNmzbdt093iE1sWtvyAk7sftjffiVAtiUdbLAlMM22nakRGMM5j867jnroarZ6FweLyR0ehdOeZYxWPXTdsywTmTb5j46s8xHPMPKBcRyNiou2Aw/R97AYZJcto1GYcBemGE/TaMEPDvF8Hg5TbiyIY/zCX/aEp/yvLoJxKA08DIOpPPpHNEon+ejA8rbjP4fReMJ+2XT9/MwsYBcXb5JMghF+2RlCb3voKsY4zT/NVlfhlBKP0SX/3k3J2c2DxeE8rfIFy3fQ1LxeT76vQuMxsa8+px8ufCe/zXMwXRZvXDxtumYkiPFyPgrpXYweunyZRGn4sAiG9OwLYToZm6SzKTkyycenaDq9wlMcZ99FCPn+zQ0dx/O04K5JbyM/ffFCz2GchqudoeJt3oV4FqbxmlxSnHUGBWULaCG3eJmXLaMYjCY7PGIMCQpojDd33lKPfCgIWIeYdneJ6Zlm3+HIadl2v22Cou4S1GFo2JDTb5mYVneJicwBT0wLtUvMgdldYjoeLzdNw2uZmEZ3iYkMVyRm21JThubD3S0ZeLtK42CYRnhODu6jRTiN5iG90OpbffIld0qe7PIxJp/G9JNlWMaFgcg/iRuEOClP8iSN8beQEX2OyZ15PhRDwTQaz8nhkJA8JOOXlNQRsbfeFCdm0WhEfwbkMY+CU9hXsAtglQfwCunilSyTOS7dXicyXz5mXMNPPfRGPnlBTj2kyxF9x55FSGQQm7xfcl0YRyHwA/Tc7YwaxiLfD8w7YuIu6JnZaky9gf5jkETD/ggPlzPKIgWzjVcDviGxj2m5Xe752maaf1hwjQlmF9VffePJBI/sDsZekriCnQHQxITEj+foIkoFl6KEKPuJfDyp2qOFK9Hi6ss9GXhDxOF6FiVEBIq0YZNoEeNhmCSHtd1jMPw2ziTjx2WayfR8vCp9nRL6lgpNE5CaEFldbfPOk8j6S/AcdEZNncqH4uxAnuoIYIOpDd0DiQ13wXz9athQTAcDMPhsiA9IlxFRwRnRrYZMl3cqIDVkNCp6K8QO6qghBaRqjRYyKQorE8dX72/b10CohLS1NBBEUW0aSNbrP4ACqs4GFjRsWQHJZgB5XuMngun/vBpWsIhYVSWka0Yg2RhoXAnZFu8LIduW5USjDoAlQ/QkLbShcgedIet4hOynbRdpIUcOrvAExykZuyfaNBifgWdolYHtfD1Dpn526dqzuhPAPJUPZ+IZItkj6b5nWH86tO4Z2hUy0tqVMvI5pTyQSDJoUkLYiv3CDYmVaKFmSSELy7v1w6f3pYpnuCZKZJRNt0OaJ59g7x8VqSK7tm/C1A6zj2WJ6AK0trXR2pJpHabBKEiJ12hcX3ZbNNZgzwGwa8uK2senJbT5J63naixZApwmDMuCpx0wyU15hp5Eiw1tO0gL2VPLvZNN2LBURTQWM7RKqHu+vgng8nXeNanBhjNxTcwf0TWpPxvad03KUuMPwzhapBI/2i498UxXKJC1ZU+GVSBzFo2vi4JnUX1icTSBI64DmSq+ttDD8VTZT2VFahyghT7zTrFJg7pMC9X54DKR2wVaQGGhogqQFQGyCqUgL0UUawR/J1qOOK/UEsxPkQfZfFcibOORayUZZYgn2qxDS/ZPu28e1uADMw+d/sDc/SuxBVl97gKASdqMR0s2WrpvPNafLKDxaPV9Y+evCbBFmy0p+6u3ye2IvHb0FGViWGRQreUEp1oKpWC3zL7t7P5FnDHlyVT2nP4BImsLoDlnYG7ayBONcEc2OKG44kAbWRRn+B2lNlaj+QRHcTTRUZrgb5YUx5dk7idtB0mhuOzDUSZNxBSRK0mSRlNEjuyt3eH5GAPJofYTck71ZHhr9Cwte80XV3U96VaqKUoBf+HxqXgHsOMaTcq5FTRGOB+9oY0cKHGnQZJEQ55D4SpK/9z5/BelInmx/Oh6VRA1O1izgzl5+j93D3a+RQ+3X8uO1qWcKSRPgpfxMNzzpgUW0yAeh/uYWkyrcMS1pdg7qRyAY2wsDqfEgX8OuYeF2Fj8wj2OaLR0Y6Dagohk057dIn/v4ltbMMg38oUbGcKNcsJIN8pQtXntE4BWQR//GEAbVAQaW5p2JkizHUVIc4yWkSZr8fpI26BmC5S/OJzAqGEI3aLyrx28wgjN1sQXj2aXwu5MUOLZvsjcPmH45o95HGY8ZlNuOskYfaP8troRVCFSXVVWGcfIKrMpWcXi0AeFFcuhnQkMbcFzQIZ9HPBsV5R6wo10Q61Cfeg5CSv9avHMkIY8oYuQebRa5G9kug2rxQrld51DGv50uUDfv3735/8LvEEaz975mLVf6RrSPNEAMwSnryrSkCjTRMjqRpqco2kMaaWo6RoaLIGJJnKOQ4NttYyGClkLEA0JebC0mo1VQ6woB9ZBxcc8+K5FHhCvrxzr2MjDgL+R7TULQO/YENePAkAm2Q4CkMW/zwSBSESgaJtXVogCAh3RW9CNQIWxr1pQIwf3YRyR56cR8bbkn1sRfmcWEHME+LlHW/4i/Bq2/D0oICYUm5H/gxlNhGTD2yzN5/DvZZikm9PsYqEYDapBo03sfg3XX4Ix3ODufbRz3+x83myP+yFhlmzTNObhjJuGZpOWz8epTNZdm+uD4suY1JZj86BYVRXm3kS0KgdiK19ICLHuOhriGb3BPe2xDV5Cqxe3V5wZGx3RuAbYOADK2rRxkUmI6lyk9H1YDrNiziO5yLMIuuLnIJlchk84DsvPv3lKgU4JJ3F4NyJuKGC3ZwiNtVm0ebdtJVDJJdob6tgNlSwdZPfvRJU/RYSoR/L7PqBYufkjiOfRvEQof1xQqR9Me87lUxDRF5kl455z3TEEqZARHg8aZAOggercBrpAA5Xx7AUNZc0Vnk7DTJXfzp/wsdAhevz37GfBs29XC/ITFJjGE/nJkoa20sN+wPOLMFdGtGDov5UK2duGhW0JsgRshA6sQnJ0lUT7kHexFxeFUXciIEoNOwaWK8IqeRlZ2wxEJm/CWQgqpgZZqE39m4YCD1Gk1VHVEns9Rs5f1JmV7FVzGIuZdyb+otDUHx3rLppiwLZhd9E0ZH/x7YpICQpEoOG5Y3yjTC1amQugPaKer2ch1725cd2e2qI+HTYCEorhHVmSsCLIRrrZm4bsDlJ4Gz8NJ8v5N8If2rbeMkzjm5IGjR3iFTIsgVnAQviBKzNL3zYRhpwo+8e5o9zKeqne5HPqtTHKcXk2efLmKB7gm2tkk+yt/WO8Mqa4htMXDV/IbrKg9ff6OCO7RNkEelWs8VwkMAbaNgwBakgjY+Sk7+tjjCtU/KBtA4vdGdOoLDPlZGhmv70qxthCLbzpyyqmWUEG9gMQeKJ7aafl9f2dPwPbE4gEpTocrkzXAIKolq0r8OFVi6JmAxRFHDndv5eYnbhIMny9oYaxtVhtT7K7/Bqud1wf8rD57cpzXyomlIZ1TYfn0gF4Vm/+6PFl4Y4cJfMa1Uiqu4+U06T6xCvvjFlpYlnaJhZUwFclp3gXJkm2a9lx2WJuceBvt9claYwYf6UbDVOyobKA9M6NXnBMAxSjKIYv/gU/UhgsH2dRmgexA8VRy7LdECtDrXYTD8TNPoP5S5t0py8hzofa3+gzQ4AecAw7xjOeLmdyx4hO2SSqGGm5/DoYU1TKbQecVO/yU06j+gYNr4JMBO2y2mRTTxPYNu00YpVuU6JuTyRHH3Rko5duiPk8pELgQwCIgMZ7nJplDTBKlXnVjZHEcm6FZJUXerwP5uO+CoHaYoPNGqzYNIyUWNFwo1OgwyaYE/3ROXHBR6gQ1L0K4ow+b0BxoxwVWoulkEA3G5DLurqamlU21W1oN/AD3KuxF0Hre4IDe3UecKq2FvE8U4ZHVn3FeEHoRIlhHrzCkq/o92XNobiG42Qus0xRxZljaRMrFdZY1po5vBogcye3ZWFdoKPeWQiUDjxoHlmQfYM26VYNO4vK4tuUCN0pf7F+Otv2xYScAUSxG00uoPJa9GQRzHtQ/DUj3kWSU49GYOc4ngVTOQZ7F6SEDzNMFw0YHzBtLDnMWuMmO7Iw/xkVpYNtz7zNcju2as6vPPU0MljByt6yMjWj7/0Qi+g2s+BMytLEtQ72sX19zAF/I6fhvj4msMHptro5AWvTqHlzOx/nK5jAc8T4Gcf5Igrg/A2OP4fPUfgCn77CtCouCxp3WcsjZyDtrCDnJDwgB6EvQgzuIiCYy7ez3DxOltR+zR2sEhBUWLF0m7zdFqPXrEXvAJMdMXQsaxMfqH5TUfP8bvXp898XSXjz/stgsDZH+NP6ihX5lvWGjMNFthmrQPK2NyGxhFQKcoCepQAZWY2OcjJ2rFVVdiRWmKvVyvvQdiY6GYntq8Tm2Me2ekFiokedTt7XamevvzvBs8dlhVi9gtnpIVtQZMRogWqugBW6LHqgft1W7RW62bK6gCir0bFBH/rtz2GQQBuH0PMPaZAuaTU4Xd2nObpTX1MNxOWa1TrMq1BVIMpV9GjrRKcQE2jVsG/en4k4dQXPREp3V25DicR4lzZxmnxNP/96+/W3P16+Ps1mlxgZX2xQnNasXzOg+jV2l4gNZKvHCxkgyoe3q+EkmI93TeBIvJFUAncw9kZ8rOh7kQOhIA2WKU4KuVEr2ragxM/Y4VzShcmg/QxG+qBsF5CvkSZOuboROraZDrBUQZOgAvEj29Q68ZN3H/i0DJeKsVIdDCKqZHAIOFPAdVHaIAMq6wZ3NlURmQMZX16wp5jxTDh8jDO+51UaKnnfXuL7NFAIQTa2ZvNAhlsbIMpLozVIgtxWfXWCwGGJcmZ2DEBBAFX/aON7eZJZMd/l3hJ67IbzhgBiiyI2TX3BLE2jU788cKpj6hMYdI7xMU7zTTXRte0okgXIEoDQtiQAVnppwkHRTuRfGNDmui4HA2b9t4eCxlzKvDtVnpLPXct/AUHsQpsvfbSBPcubVQ+bneqbkguv1TbY9G3Zt4CyUS8RXKvRkHX46rwDW/QOPGDm6yrfgdlf3mRSz8zvFs81SH/k8O4BYA7o2u0PRgBUn6rVN+AgAK4AfXWgcEVXAUhvQY0l9aGisejhjkXQLdmgQhaYvDlgeYA50Kxv0FiQUPYNusV+LQEDIWKMADgoMg/IYYwphzbn3tHqqDs8CukV/wc= \ No newline at end of file diff --git a/docs/extraction/Extraction-Pipeline-v2.3.png b/docs/extraction/Extraction-Pipeline-v2.3.png new file mode 100644 index 000000000..6addc880d Binary files /dev/null and b/docs/extraction/Extraction-Pipeline-v2.3.png differ diff --git a/docs/extraction/Extraction-Pipeline.drawio b/docs/extraction/Extraction-Pipeline.drawio index 262b42da3..743779856 100644 --- a/docs/extraction/Extraction-Pipeline.drawio +++ b/docs/extraction/Extraction-Pipeline.drawio @@ -1 +1 @@ -5V1dc5s4F/41ntl9Z+JBiA9zmThNN7tJmzZ9u9urHWITm62NXMCJ3Yv97SsBso10sDFBYJLmokZgDM85Ot866uHhfPU+dBfTWzL2Zj1dG696+LKn68hwdPofG1mnI7ZhpQOT0B9nF20H7v2fXjaoZaNLf+xFuQtjQmaxv8gPjkgQeKM4N+aGIXnOX/ZIZvlfXbgTTxq4H7kzefRPfxxP09GBqW3Hf/P8yZT/MtKyM3OXX5wNRFN3TJ53hvC7Hh6GhMTpp/lq6M0YeByX9HtXBWc3DxZ6QVzmC7pj4hm6XE9/rjztITKGn+MPZ46Z3ubJnS2zN86eNl5zCEKyDMYeu4vWwxfPUz/27hfuiJ19pkSnY9N4PqNHiH589GezIZmRMPkuxthxrq7YOAnijLqI3UZ++uyFnrww9lY7Q9nbvPfI3IvDNb0kO2sOMmQz1sJW9jLPW0JxNpru0IgTxM1YY7K58xY9+iED8Bgwje6CaSPUN3Nw6obRbxtQ3F1ATc4NGzidlsHUuwsmRoM8mDpuF8wB6i6Ypp2Xm0izWwZT6y6YWLNEMNuWmjJr3t9e04F3qzh0R7FPAnpw5y+8mR947EK9T79izehzXTyE9NOEfdI15JwhdKbbEi0oNHEe8CgOyXePQx4Qet88FbIhd+ZPAno4ooB7dPyCAe1Ta+s8OzH3x2P2MyCF8zzwEuJlxAIIZQGUwqooJUvkHI2uLyOZLh8TmpHHHj6XT57RU/fxcszesadTiDRqkfcLrvNC3wN+gJ27njOzWKT7gVlHDdwFOzNfTZgv0H9wI3/UH5PRcs5IVMNcyysBR5PIx3XcLvUcZfPMOSy2JpRnF+VffePHuA/8DtpeSCzBygAwQZDwsU1VoJRwKApA2Q9ydajaw8KSsBh+uaMD51Qcrud+REWgiA2fRIuQjLwoOqzrHtzR90kiGT8u40Sip+Nl8TUL8C0UmgiQmhCslrJ5Z0uw/u4+uZ1RUy+lQ3Z2IE91DJABKePugUSGWzdYvxkyZNNBA8w9A6IDVmVElHBFVKuhjdm0Rw1pjYreEpGDY9RQDVC1hoUMRWZlknB4c92+BsIF0B6lgSBElWkgWa+/AgVUngw8ZNiyApLNAPq82i+Up399M6Tg8bCySkjVjMCyMdC4EjL0vC+EDUOWE406ALrMoi/SQhuUO+gM6dU5ZD+2XcRCjhwMyZSEMR27o9rUnZyAZ6gXMdvpeoZc/ezi2uOpgq5qgyPocCKeIZY9ku57hsdPh9Y9Q6NEPlq5UsZOTikPJEgGTUoIo2a/cANxLVqoWShkYXm7vv90U6h4RmuqRMbJdDukedIJdvNQkyoyjvZNuNrh9rEsEaFEj6EMa13G2ovdsRtTr1G7vOi2aDyCPAeYXVlO1KiellDmn7Seq9FlCfAyYVgUPO2ASY7kGfoiLDbYdhAL2VNLvZNN2LBQRTQWM9QL0D1d3wRw+TrvmhxBhhNxTdBrdE2Onw3tuyZFqfH7UegvYokebZee2MgSymMN2ZPh9cc5i8ZRheBJVJ/oOUzgiOtARsVRFnqojsp+lGtS4wAW6sy7mk0a3GUs6s4HF4ncLmABhYWyKkBeBMgrlNy0FFGsEfxKtRx1XpklmJ6iD7L5rgRs45HrWjLKEE2UWYe67J923zw8gg7cPDT7A7T7J5EF6/3cBQCRlBmPumy0dN94PH6ygMaj3ne0nT8EkEWZLSn7q9fR9Zi+tv/oJ2JYJNBRiwleaikUMruO+oa5+4dzxpQto2yb/QMgKwugmSdgboq5hE1A7UBUcaAMlJrz+2atFlaj2QS+ILY2KGpN7zcLRfWCzP3QdhCKmos+zNpkiZggsiRJ0miCyJR9tVsSTAiQGmo/HWeWT4W3hmdh0Wu6tKrrKbdCTVHI8Gd2XnmagBXXaErOKqExvGB8zpo4MHBnbhT5ozyFvJUf/7Xz+RtDkb5YenS5ykBNDtb8IKBP/9fuwc632OH2a8nRupAymeSJyDIceXveNOPF2A0n3j6iZtPKG+daUuydVCZAMT4WejPqvj95uYeFyJj9wh3xWax0Y54agojk057fIn3v7FtbZpBv5Ag30oQbpcBIN0q4avPaL2C0Evr4dTDaoCSj8YVpJ8JphlkTp5lay5wma/HjOW3DNVtG+ZbjE5hrOIduufLbDr/CHJqsh88ezShkuxPhEttwROL2KcE3/1A1nrG5TbnpIqP1teLbquagEnHqsrJKqyKrUFOyivvuB4UVz6CdCBsagueANaMa4xmWKPWEG6lmtRLVoackrNSrxRPjNGwLHYRQZbWYvxGyGlaLJYrvOsdp5NPFAv/856cT/O3agzicv3cIb73SNU6zRQNME5y+spyGRZkmsqxqTpMzNI1xWiHXdI0bdIGICJvVuMHQW+aGEjkLkBsi+mBxORvrCLFSO2MdVHzcg+9a5AHn9ZWpV408DPI3MuxmGdCuGuJ6LQzIJdtBBuTx7xPhQCxyoGibl1aIAgeaoregmgNrjH0dxWr04M4Lffr8LCLelvyzSrLfiQXETIH9rMqWv8h+DVv+NhQQE0rN6P/unCVCkuFtluaz92PpRfHmNL9YKEWDKtBYC7s/vPUXdwK3t7vxd+6bnE9b7eV+SJgl2zQNOpxxU9BoUnfycSqEgFpbzZF5UlmOzYZiVWWIe+WzmhyIrPkyQoh0l/6IzNkN7lh/bfASVru4veLEyGiKxjVAxgFQ1KaMilxClKciw/d+OUpKOStSMU8i6Irf3Gh64T2S0Cs+f/4YA30SXkTh3Yi4VgO5bU1oqs2jzbtNK4E6LtHeqI/cUMnSQXJ/par80aegVqT3nct45epPNwz8oEAof1wwqe/OeubFo+uzF5lHk5552TEOqkNG2HmmwQbANFCd20AV00BlPHuZhpFmSGYzL1Hl18Ejqco6VI9/TX4WPPtutaA/wRhTe6Q/WdDOVnrYDyQ481JlxAqG/leqjL1ttjB0QZaATdCBNUimqoJoB/Iu9vJFZtS9kCEKDTvOLENKKnkRWdsExChvwukYKqUGSahM/SOtBg9RxKpStcRejzHnL6rMSvbKOYzZzDsRf1Fo6I+ruotIDNg27C4iTfYX362olGCMCLQ7N7XvjKhZI3OBaSvU8/V0bFlXV5bVq7eoT4WNgIX1qKYsSXgRZCO97JEmu4OMvbVfRtNl8J3ShzWt1zWkfa+lPWOHaIU1XSAWsAx+YMnEUrdFhCYnyv41bxm1kk6qV+mcemuEMq08mWx5YxQb8M0Vkkn21v7V3hhRLM3si4YvZDfp0Op7dZSRXaJkAr0p0tgWFggDbRmGATWkkDBy0vftEcYSKn7wtn3F7oxpVJYhORma2G9vijCGUAuPHFnFNCvIwG4AAk1UL+zU7b6z829g2AJIUKrDzJXpakAQVTdUBT7sclHUZIBxUQ5O68eS8BNnUcJf58ww1her7Ul+lz+89Y7rQx82vV1x7quOCaVgXdPhuXSAPcu3frTzZeGmHCWzG9VIdfceKcak/MQr7otZamLpyiYWVMBXJqd460VRsmdZtWxxbnHg/68vC9IYIfmHbTLMYMNFAemdGz2TkAUoxn4IX/w7eWBssHyY+3EaxHZrjloW7YRYmtWObuGBc7NP4/7SJt3pSBznQM1v1JkhQAc4zjvaE5kt53K/iE7ZJHURUrfy62CQqJTbDjjVvcdPMUbHGzR5FYQwtMNqky09EbBp2svAKtykpL4dkUx1rCMbvWw7zKcREwIfXEAENN7hFBU1wChU5mW3RRLLuWuEVV7oceMGk34dArXF9ppHkGLTLlIiRcNtToH+mmBO9LVT4iwfocJlG5+q8wZqbpRTh9biKSTQzQbksqqepqjMlroN7QR+gHpH7ETQ+n7gwE6dB5yqrUUcJMqwYtVXSBYUJwYGOniFLl/R78uao+YajhdTmWeKSs4cXZlYKbHG8qiZk1cDdO6ktiysC1TUOwuB0oENzSMdmkjqdh4GNphCEsyd8haPT2YbjpiO04AYdqOpBVxciR4t3KAHRV8T8M6iFD0Wfw1IOHdncgT21o0pHeaELRnQPhDWVHKUtMWNdiRh+jN1FA62Pe/MARI7nfMqoIMTTx2Ba1jXW1SkpvXtV7GEbjMLTqQoTVzpYFTt6oMG+RuZDXf1QcDmptva5gisTGPGzXUwSdcvgeeo6TMJ0yUUwPkrEn72nnzvGT49JKwmLgkZd1nHY6F+wDAgJW8DGQh18WFwBwHBWL6ep8ZxtGTWa+peFTBBifVK19G7bSn6kZXoHSCyKQaOZQo7QO1bHRXP71efPv84i7yrmy+DwRqNyaf1kJf4FnWGDL1FshGrAHnbG5DoQiIFm0DHUgBGPsNqh7FjjaqSI7G+vF6tvI/bTkQnY7F5ldgau2qjFyymeerTyfsa7ez1dqdk/rAsEakvmIoAYxTHe7EhGM3UaIEqroD1uTx2UP+qraPX5yaL6lyqrMZVQz7s2589N4I2DWHn72M3XrJacLa2T3Fsp4LrIy7WLNdfvg5VBXJ5HR3aGu8TIoi6avIVAZ0bigVBW+LUEjwTKdldugklFqNddYlTehgSFs/YXs4MlFsy9tgV/wE= \ No newline at end of file +7V1bc6M4Fv41qZrdqrhA4vrYSTqzmel0J53unsvLFrGJQ9o2HowTux/2t68EyAbpYMCWwCQ9XbUbCyzwOZ/OXUcn+Hy6+jXy5o/X4cifnCBttDrBFycIuY5O/pcOrNMB07LTgXEUjNIhfTtwF/zws0EtG10GI39RuDEOw0kczIuDw3A284dxYcyLovCleNtDOCk+de6NfWHgbuhNxNE/glH8mI46yN6O/8cPxo/sybrlplemHrs5+yWLR28UvuSG8PsTfB6FYZz+NV2d+xNKO0aX9HuXJVc3Lxb5s7jOF27XXx5Xlv/976ev/tpd3HxbPixPsZNO8+xNltkvzt42XjMSROFyNvLpLNoJPnt5DGL/bu4N6dUXwnMy9hhPJ+STTv58CCaT83ASRsl3Mcaue3lJx8NZnHFXp9Nkj/Wj2F+V/iB9QyYCLz+c+nG0JrdkX7A0PDDTL2XgMnBG/JctqxiQHnNcYizxMnCMN3Nv6Uf+yEjYhJx2f8lpMCxkxMS6xcjbAjmRa+KJfrF+/LHytfuFcf45/njqmp2SU6AdQOFydFo2j04Ld0xO3F9y2qZeICYy3c7RifpLTqw7HDoxQl0TlNkKfSSoaWkFcuqa3Tk5tf6SE2vWsZFTBOfd9RUZeL+KI28YB+GMfLgJ5v4kmPn0RjTAA/Iy1oS82dl9RP4a07+QhrRTzT3VdIEbhDhxkeSLOAq/+4zos5DMXORDNuRNgvGMfBwSkvtk/IySOiBm7LvswjQYjehjQB4XUXAI+5iVL7LKBniFVfFKlMwFLl1dLES+fEq4Fj6c4HfixVNy6S5ejuhvPEGERBrxdAYl9/lR4AMPoNeuptTf4Plese6I5zCnV6arMfWxBvfeIhgORuFwOaUskrDaipafqwnsY7ouzz1X2UpzqwXXmGB2Xv+nbxxE757NoO0kiWXzzoVuiWTRIQlkm6roUsMcLqHLbjrvT63uaGEJtDj/ckMG3hGJuJ4GCyIFedqwdTSPwqG/WFQrvHtv+H2cCMdPyzgR6+l4XfqaJfQtlZs6IDghslrKlp7owP7mPXu90VSH8iG76ohLHQNs0JWhWwzLXHuz9ZthQ7YcNMDmMyA+YFV2RA2PRLUm0jnPAtDOutaq6K0RRmiihiSQqjNaiKTIDM0wOv9w1b0GwiWkbaSBIIoq00CiXn8FCqg+G1i4q2MFJJoB5H21Xwim//VmWMEijXWVkKoVUSdHo1oJGcjgo9kGhiIyrToBqEa6pYkm2lC6hw4R2h8lu2nbR1qIAYTz8DGMYjJ2QzSqNz4C7xCVge14vUOmgvJ0PUG43xqhAR+OxDvEolfSf++w+XLo3Ds0jCNQzKzkISOKI5DEaVNCGJJ9ww2JpWihdkkhCsvr9d3th1LFM1wTJTJKlluV5kkX2Id7SarIaOyfMLXDbGRRIloArQ1ltEYirf3YG3kx8Ry1i7N+i8YG7KkAu7LkqLF/akKmj1IAJRQoa9cMrZGAbyQMywKoPTDJdXGFHkSLDW17SAvRU0u9k03osFRFtBY3RCXUPV7fBHD5eu+aNGDDkbgm+mt0TZqvhu5dk7L0+N0wCuaxwI+uK1BM2+TLLVjJSd6mNEUSIksVCY+gCsV26oZdHZEyrrL4w/6U2U1pSbocoIU6G0+yXYP7TAvZieEyudsHWkCxoawikBUEslIlLy1L5OsFvxFVRzxYag6ml8iLbL4rELb18LWU1DLEE2UmIhKd1P7biA34wGxEc+Do+X8CWzAaFG4AmKTMgkSi5dJ/C7L5YgEtSDRwtdw/HWCLMoNSdFqvFlcj8rODhyARwzyDGm0tONRSKAU70geGmf+HC+aULVKZWKIVRFYWRTOPwOTktwGaJmRwQsFFRxlZJKf5Tak2VqtJBVNySNGUmuVvlxT712buJm0PSSG59sOUJk34PJElSJJW80Sm6K1dh7NxCGSIus/KmfUz4p3Rs7T+Nd1o1ffMW6mmKAX8qV3Mx5s1lacym8KqoTH82egd7ZVBiTvxFotgWOSQvwriP3N//0WpSH5Y+ulilRE1+bBmH2bk7f/Mf8h9i37cfi35tC7lDNsnGS6job/jl2ZYjL1o7O9iaras/FGh88fORWUCHGNjkT8hDvyzX3hZiI3ZE27CgIZMNwaqwYlItuzZFOnvzr61BYM4kctNpHETpYQRJkpQtfnZBwCthj5+HUBzagLNxkeFNMOUhDRT6xhpNeKMdZGm7YM0vTWk2TWhxipejwVqusbnLbbNeprCzbD4ybApTKYacjVK/SohtxFUW8T8VQAMDJ8NVDfw/CsHYhiq6oXbkSEO28XiH6TvLdyKE+lWy8JNjH+2hrRS1PQNDYjvxoL3lT2IU3U8rFSjoUZEEETDgrxYXE8DNhAr0oFVKY6Yddw3qx4XpYiJ9rXqneJEht0uAO193cfXAkAm2SoByGJLR4JAjPnSEUu0muqiEAt1KJZoz6lGogz/snVfkRViVcKHNTQ5EvhYfFhCs/eDDt8YzeKDkaphgwDYcIUY5P+9KQ0SJsPbCOZn/5+lv4g3l9nNXKEGVJ9Bmz397q+/eGO4EdSHIDdvcj1tSlV4EIfubQhTr45GK2jKhlgylFlVeqabCs0CXBGTyuLPNhQJqMPcy4BmrCG2FotsINZdBMNwSie4oS1+wVs+LeP5Mt7ec2SMNE2h2hIoznGAsg9lnHSgFNZOTlLqnoeTiZ8s1avZQ7gvQ8k6/ZY8Frz6fjUnjyDcQtoDeWRJYzfhZT+Gs1M/BRtNlv27VhFX18AwEA+MsragQCWuqaokyIU0/05sZIL7QFCUCm8GmHPCLrGWumsmYl3nmcj2AFSxUNny1jUJ1htPq72yBeTDjR8F5GfRHGJHGQRU0yg8rgSCXbTk8L4hNp0PqrQcYtM10SZ8vyJSggIRaP5pat8pU7O2nhxo98hnnyBsWZeXlnUiN6mtwlLgu2CaojJgRQCtdHbVNdHko/DWfhk+LmffCX9oC1ek6dp3KZ2KesQrrCGOWcBuMMcSmaWuZbImBrP/Z15TbiVNxS7TNfXWGMW7xACbbMD2Vsgm0fhO2PSmuGK5Lrd8kA2dgIEBcaeQNWIC4O2xxuaCUYYJlVWhVteMLgbGEzvhTTHGcPnUPHKhNYMAX0Mda8C9VxxfVJfRI3vg5v5zDLtAJt2FQmfmQMv/B209MFQ52TZUZC862ckARVKBnNY/y5BdOF0kGHtHjTA0X20vslmI95wzs8nLptOVx1JlLCoFNaTV66kCnvW77dgDM48LZnHnDYZWtZLsnZ7lNKm/8MpbEdVaWEjZwoIKOurEqK/9xSI5LWK/7EOhEPvr1QUcyrqJwid6bh4lGy4LgOYmegkj6gyPggi++bfwnsJgeT8N4jRo6kmOkJWdQlMbao03TOLC6tPs4gYpnR2jkD8uA9pqrM4UAZpuMOxoz+FkORV35/XKLpHFSGQV5ICu80q56+CG7Nbq5TRqbtAUVZDOjjrsrg09cFbFYcQq7Q0trxG9qQ46otFLDyJ6HlIh8NEDREDrTaX0ss2Gpcq8bjd6vrxPIlnFngEfvNl4IEOgdtjRqAErmA8OtAlot7MU0NIIzL+9dk6cFrewQ9uzW+1Pv0nnHJPWYukK0M0G5LKrDLY1KnhbOoWxgnsNmr92fhYjcEBShVO1tYhniTLcryqAuEtzQidKDL3yDiTeMRiImkNyvcDBXGatMWquHKRMrMg+/bmoBsjaSW1ZWBeoqJ/jNiU44JmmCLJvMOKrbeXRGejqrwuE7pW/2Dx1ym2NxBoYxW41wQAc+7Itd1uUnlJ6NRunJavgNSKdxhE19sHrl2H02X8O/Bf48nlISySSqE6flyFmyp7ZCDZ0lLgNBAnVhXDAloqcPsvOjtUWS6pgUguoBAQ1SlSvFu+31YkNixN7wGSTj+2IHHaBUggZBXC/rm4//3O68C8/fHGctT4Kb9fnVScSR/48OZ6GI3nXbVmRzR8TD6deLYCUloRWZCApe7b7O/nElxyW8mav/Ui7EHcktYNY3BG+FbuNtyMJO8INJEwmr4gwvD2b4x9PP9zZfz3biaPpr24IHAcvCszHcHq/rBFYk7BSba7vDDbEHCGDTsF+scs5WXeVLp7iz79fPX394+XpYTo9C7H2xQDpkyqZxdybFQhVlj3WoOwxmyVgA5kc9UfZng5O193FXrwkum746M3GeUUX8DPSerTk1WpnnYklFfzIQhF0VXvLOFxkeqyR0TunKE04YJ6dmBewlgQN7qqgU2MgOVoxvYURZAq7gKSQoTtBKJUnUGsXItSFUoqST9Ht0l/6adR8J2CKpQuH+0ytBCKbl8Zx5VesXrsi4ijDNwIBUR78kQwIcY9RTUHSEBf1+c2LHFFycEJIAvsxK1batMQAAxetQgAov1OEgWwvUe8YH4Vx2lgcXximpAAWy8GwECFUa9MqCkS3Si0K3qoIQBbvCG46oVSd+KOO+eWbf5WrgcQ+eEsAMHS+Oh7btcPX6iBQfsqCmvXfL74rUAHYLNoCgApQ1d4URgCUJFLkHGw6B9CDOwpIAKsxe4MNWV6Cw0dlNm23uhMQrfmOOQOhX0JChlDQeesA2+AxVaqMw9v1l8eV5X//++mrv3YXN9+WD8s6EbpcVUrtWhOAVjuCKVzK1wIsZlVnT4JEgQxm6UQpZ8fuislWKSFaj6mZNw/8qFyuC2u5rhhxICnySzibUCI80EOeyOvmj1PR/E2CiO6QrdotE5RKE8l1ngAmKnmPYN53cHYo+HqiFXn050JJ5MJGVglcUFXgCb6daMgd/blP8pdCmweHgq8no/ts630duRxnZWIV1k91zw9IbcquEqmuhYqWlsYdgFq7C4umFWfaZPHlZ1BhgrcWvMu5bsmRZa/Sd6sQRhXpY7dDz23ncpSYYD+MQjbfVcsVFWarGfbykjF1y6d3sW/5YQ4b82EOKA2iKhIOr5UaldLHd/RKU629swNoXmvvkCddaW1dw1wkgK+WqK+2neJMLjdRidYmzPfWuduylVP6xhbX7U3PzkXY4jSdUa5J8NP8rARyxzjmKuV1/tTP2jjW+dgYv61asfmJasQH+4+1nb3G81grL3LsCmtC1ydtT6jZ3MlomN+FrRppkKPz6pC2K8h65EDjVJ3Bb5WtCzSH2+GF+Yilao+6Xg8nsdVMWjyblsqe7Lk1Mu15nuCkbM/J5gE7rmlTtlGTd/YOahpDP2dzle2EbOQN8scjWHCbbBsArVGOz8NSO63Vw3z+Pb29cGsRZt6M7msBscN/5Wf8hebOuaMVWEOs/I60Nl1KsIeeEjCxgd3bAl6yH00nnoXR1JuIU5fDkivtTxJt8p5b+bzXA/W8HC3vldggQc7viW4R9ztP3NxThX7zo+CBUD7d1rdnjwGP7s+9/MOLZsGs5ASCT3P6BAJG8+zBC+ivmi7GlIE7VPNxHkXC2eZ0f6ENaFIoFWmZ/J4veeJPZiDiZI+I2l5HzEpxDtkKrD5xz+7UaOe9Q2fPPJgpHJRmOI66zYQw3MqljppYfl5a+aM+R/Vzha2HWfR60QIzMWjRQ4hWZ4S1ttEJhMUrSpPKQomJ+ErHEpyo2gwF46RGj6n+B5h2bcOuPqDY6FJZubyVs2+ESdeMbkNMQLOgn1CDOjR0BTVHK7YrRkW4OFqxBe5+MORPBxVC86pRWN5doP1Q1yx8pgpz/TPcVV+Nuvy5UtiENgyo2koEgwodEah+QurASBK8PbHVGCpuLYZaMN8/hokBv35F5rsssePwu5SAyq1WLXfc9vbFnIf3EyDVQgSo8QRDkeoAApygp9Ug+5F15zxs2drCqcQuZC1I6s9JPkYhXUxbi5V217sORz694/8= \ No newline at end of file diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index abfe624aa..9520fe819 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -7,6 +7,6 @@ [assembly: AssemblyCulture("")] // These should be overwritten by release builds -[assembly: AssemblyVersion("1.11.1")] -[assembly: AssemblyFileVersion("1.11.1")] -[assembly: AssemblyInformationalVersion("1.11.1")] // This one can have the extra build info after it +[assembly: AssemblyVersion("1.12.0")] +[assembly: AssemblyFileVersion("1.12.0")] +[assembly: AssemblyInformationalVersion("1.12.0")] // This one can have the extra build info after it diff --git a/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd b/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd index 09b59dbe6..e6e96f01f 100644 --- a/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd +++ b/src/applications/Applications.DicomDirectoryProcessor/ProcessDirectory.cd @@ -1,61 +1,78 @@  - - + + + + AAAAAAAAACAAAAAAAAAAAAIAAAAAAQAAIAAAAAAAAAA= + Execution\DicomDirectoryProcessorHost.cs + + + + + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAACACAAAAABAAQAAAAAA= + Options\DicomDirectoryProcessorCliOptions.cs + + + + - - - - Execution\DicomDirectoryFinder.cs - - - - AAAAAQAABAAiAIACAAAAAAQAIAEAAYAAIAAAQCAAAAA= - Execution\DicomDirectoryFinder.cs + AAAAAACAAAICABACAAAgAQgCAAAAAIIAcAAAQCCAAAg= + Execution\DirectoryFinders\DicomDirectoryFinder.cs + - - + + - AAAAAAAAACAAAAAAAAAAAAIAAAAAAQAAIAAAAAAAAAA= - Execution\ProcessDirectoryHost.cs + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBAAAA= + Execution\DirectoryFinders\AccessionDirectoryLister.cs - - - - - - + + - AAAAAAAAAAAAAAAEAAAAAAAAACAAAAAAABAAQAAAAAA= - Options\ProcessDirectoryCliOptions.cs + AAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAACAAAAA= + Execution\DirectoryFinders\BasicDicomDirectoryFinder.cs - - - - - - - - + + + + AAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACBAAAA= + Execution\DirectoryFinders\PacsDirectoryFinder.cs + - - + + - + - + + + + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAACAAAAA= + Execution\DirectoryFinders\IDicomDirectoryFinder.cs + + \ No newline at end of file diff --git a/src/applications/Applications.DicomDirectoryProcessor/Program.cs b/src/applications/Applications.DicomDirectoryProcessor/Program.cs index 05bc8ccd4..4d04a590f 100644 --- a/src/applications/Applications.DicomDirectoryProcessor/Program.cs +++ b/src/applications/Applications.DicomDirectoryProcessor/Program.cs @@ -26,7 +26,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( processDirectoryOptions => { - GlobalOptions globalOptions = GlobalOptions.Load(processDirectoryOptions); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load(processDirectoryOptions); var bootStrapper = new MicroserviceHostBootstrapper(() => new DicomDirectoryProcessorHost(globalOptions, processDirectoryOptions)); return bootStrapper.Main(); diff --git a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj index c9d009e11..3c9a85ffe 100644 --- a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj +++ b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewer.csproj @@ -10,6 +10,7 @@ 8.0 full true + true @@ -33,7 +34,7 @@ - + diff --git a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs index 1432379de..f258cb528 100644 --- a/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs +++ b/src/applications/IsIdentifiableReviewer/IsIdentifiableReviewerOptions.cs @@ -3,6 +3,10 @@ namespace IsIdentifiableReviewer { +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + /// + /// CLI options for the reviewer + /// public class IsIdentifiableReviewerOptions { @@ -47,4 +51,5 @@ public class IsIdentifiableReviewerOptions )] public bool OnlyRules { get; set; } } +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member } \ No newline at end of file diff --git a/src/applications/IsIdentifiableReviewer/MainWindow.cs b/src/applications/IsIdentifiableReviewer/MainWindow.cs index 33ff4a260..419eac03f 100644 --- a/src/applications/IsIdentifiableReviewer/MainWindow.cs +++ b/src/applications/IsIdentifiableReviewer/MainWindow.cs @@ -15,16 +15,41 @@ class MainWindow : View,IRulePatternFactory { private readonly List _targets; + /// + /// The currently selected database which will be updated when performing redactions (when not operating in Rules Only mode) + /// public Target CurrentTarget { get; set; } + + /// + /// The report CSV file that is currently open + /// public ReportReader CurrentReport { get; set; } + /// + /// Generates suggested ignore rules for false positives + /// public IgnoreRuleGenerator Ignorer { get; } + /// + /// Updates the database to perform redactions (when not operating in Rules Only mode) + /// public RowUpdater Updater { get; } + /// + /// Width of modal popup dialogues + /// public int DlgWidth = 78; + + /// + /// Height of modal popup dialogues + /// public int DlgHeight = 18; + + /// + /// Border boundary of modal popup dialogues + /// public int DlgBoundary = 2; + private ValuePane _valuePane; private Label _info; private TextField _gotoTextField; @@ -34,8 +59,19 @@ class MainWindow : View,IRulePatternFactory private Label _updateRuleLabel; private CheckBox _cbRulesOnly; + /// + /// Record of new rules added (e.g. Ignore with pattern X) along with the index of the failure. This allows undoing user decisions + /// Stack History = new Stack(); + ColorScheme _greyOnBlack = new ColorScheme() + { + Normal = Attribute.Make(Color.Black,Color.Gray), + HotFocus = Attribute.Make(Color.Black,Color.Gray), + Disabled = Attribute.Make(Color.Black,Color.Gray), + Focus = Attribute.Make(Color.Black,Color.Gray), + }; + public MainWindow(List targets, IsIdentifiableReviewerOptions opts, IgnoreRuleGenerator ignorer, RowUpdater updater) { _targets = targets; @@ -66,7 +102,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno Height = 1 }; - _info.TextColor = Attribute.Make(Color.Black,Color.Gray); + _info.ColorScheme = _greyOnBlack; _valuePane = new ValuePane() { @@ -101,7 +137,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno X=28, Width = 5 }; - _gotoTextField.Changed += (s,e) => GoTo(); + _gotoTextField.TextChanged = (s) => GoTo(); frame.Add(_gotoTextField); frame.Add(new Label(23,0,"GoTo:")); @@ -133,7 +169,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno frame.Add(_updateRuleLabel); var cbCustomPattern = new CheckBox(23,1,"Custom Patterns",false); - cbCustomPattern.Toggled += (c, s) => + cbCustomPattern.Toggled = (b) => { Updater.RulesFactory = cbCustomPattern.Checked ? this : _origUpdaterRulesFactory; Ignorer.RulesFactory = cbCustomPattern.Checked ? this : _origIgnorerRulesFactory; @@ -143,7 +179,7 @@ public MainWindow(List targets, IsIdentifiableReviewerOptions opts, Igno _cbRulesOnly = new CheckBox(23,2,"Rules Only",opts.OnlyRules); Updater.RulesOnly = opts.OnlyRules; - _cbRulesOnly.Toggled += (c, s) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; + _cbRulesOnly.Toggled = (b) => { Updater.RulesOnly = _cbRulesOnly.Checked;}; frame.Add(_cbRulesOnly); top.Add (menu); @@ -542,11 +578,13 @@ public string GetPattern(object sender,Failure failure) var recommendedPattern = defaultFactory.GetPattern(sender,failure); Dictionary buttons = new Dictionary(); - buttons.Add("Clear",""); - buttons.Add("Full",_origIgnorerRulesFactory.GetPattern(sender,failure)); - buttons.Add("Captures",_origUpdaterRulesFactory.GetPattern(sender,failure)); + buttons.Add("x",""); + buttons.Add("F",_origIgnorerRulesFactory.GetPattern(sender,failure)); + buttons.Add("G",_origUpdaterRulesFactory.GetPattern(sender,failure)); - buttons.Add("Symbols",new SymbolsRulesFactory().GetPattern(sender,failure)); + buttons.Add(@"\d",new SymbolsRulesFactory {Mode= SymbolsRuleFactoryMode.DigitsOnly}.GetPattern(sender,failure)); + buttons.Add(@"\c",new SymbolsRulesFactory{Mode= SymbolsRuleFactoryMode.CharactersOnly}.GetPattern(sender,failure)); + buttons.Add(@"\d\c",new SymbolsRulesFactory().GetPattern(sender,failure)); if (GetText("Pattern", "Enter pattern to match failure", recommendedPattern, out string chosen,buttons)) { diff --git a/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs index 912510260..afebf0f91 100644 --- a/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/IRulePatternFactory.cs @@ -2,6 +2,9 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Interface for classes which generate patterns for classifying as true or false positives and describing which part to redact. Can be a strategy (e.g. use whole of the input string) or involve user input (e.g. get user to type in the pattern they want). + /// public interface IRulePatternFactory { /// diff --git a/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs b/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs index cd46f8174..4db1e9d0a 100644 --- a/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs +++ b/src/applications/IsIdentifiableReviewer/Out/IgnoreRuleGenerator.cs @@ -5,14 +5,31 @@ namespace IsIdentifiableReviewer.Out { + /// + /// + /// Implementation of OutBase for . Base class should + /// be interpreted as rules for detecting which are false positives. + /// + /// See also: + /// public class IgnoreRuleGenerator: OutBase { + /// + /// Default name for the false positive detection rules (for ignoring failures). This file will be appended to as new rules are added. + /// public const string DefaultFileName = "NewRules.yaml"; + /// + /// Creates a new instance which stores rules in the (which will also have existing rules loaded from) + /// public IgnoreRuleGenerator(FileInfo rulesFile):base(rulesFile) { } + + /// + /// Creates a new instance which stores rules in the + /// public IgnoreRuleGenerator() : this(new FileInfo(DefaultFileName)) { } diff --git a/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs index 0197478ce..0b82828f2 100644 --- a/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/MatchProblemValuesPatternFactory.cs @@ -1,14 +1,24 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using Microservices.IsIdentifiable.Failures; using Microservices.IsIdentifiable.Reporting; namespace IsIdentifiableReviewer.Out { + /// + /// which generates rule patterns that match only the and allowing anything between/before + /// public class MatchProblemValuesPatternFactory: IRulePatternFactory { private MatchWholeStringRulePatternFactory _fallback = new MatchWholeStringRulePatternFactory(); + /// + /// Returns a pattern that matches in . If the word appears at the start/end of the value then ^ or $ is used. When there are multiple failing parts anything is permitted inbweteen i.e. .* + /// + /// + /// + /// public string GetPattern(object sender, Failure failure) { StringBuilder sb = new StringBuilder(); diff --git a/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs b/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs index ecf73b8d7..27c108767 100644 --- a/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/MatchWholeStringRulePatternFactory.cs @@ -3,8 +3,17 @@ namespace IsIdentifiableReviewer.Out { + /// + /// that generates Regex rule patterns that match the full (entire cell value) only. + /// public class MatchWholeStringRulePatternFactory: IRulePatternFactory { + /// + /// Returns a Regex pattern that matches the full cell value represented by the exactly (with no permitted leading/trailing content) + /// + /// + /// + /// public string GetPattern(object sender,Failure failure) { return "^" + Regex.Escape(failure.ProblemValue) + "$"; diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs index 59f023445..37620b0a9 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBase.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBase.cs @@ -11,15 +11,35 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Abstract base for classes who act upon by creating new and/or redacting the database. + /// public abstract class OutBase { + /// + /// Existing rules which describe how to detect a that should be handled by this class. These are synced with the contents of the + /// public List Rules { get;} + + /// + /// Persistence of + /// public FileInfo RulesFile { get; } + /// + /// Factory for creating new when encountering novel that do not match any existing rules. May involve user input. + /// public IRulePatternFactory RulesFactory { get; set; } = new MatchWholeStringRulePatternFactory(); + /// + /// Record of changes to (and ). + /// public Stack History = new Stack(); + /// + /// Creates a new instance, populating with the files serialized in + /// + /// Location to load/persist rules from/to. Will be created if it does not exist yet protected OutBase(FileInfo rulesFile) { RulesFile = rulesFile; @@ -86,6 +106,9 @@ protected IsIdentifiableRule Add(Failure f, RuleAction action) return rule; } + /// + /// Removes the last entry from the and . + /// public void Undo() { if(History.Count == 0) diff --git a/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs index bdad48d04..a2bdb64d1 100644 --- a/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/OutBaseHistory.cs @@ -2,11 +2,26 @@ namespace IsIdentifiableReviewer.Out { + /// + /// Record of a rule added to during the current session and the serialized that was persisted + /// public class OutBaseHistory { + /// + /// The rule generated + /// public IsIdentifiableRule Rule { get; } + + /// + /// The serialized representation of the (added to ) + /// public string Yaml { get; } + /// + /// Records a serialized + /// + /// + /// public OutBaseHistory(IsIdentifiableRule rule, string yaml) { Rule = rule; diff --git a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs index c47b29b79..22b8e6c9e 100644 --- a/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs +++ b/src/applications/IsIdentifiableReviewer/Out/RowUpdater.cs @@ -10,8 +10,19 @@ namespace IsIdentifiableReviewer.Out { + /// + /// + /// Implementation of OutBase for . Base class should + /// be interpreted as rules for detecting which should be redacted. This involves adding a new rule + /// to redact the failure. Then if not in updating the database to perform the redaction. + /// + /// See also: + /// public class RowUpdater : OutBase { + /// + /// Default name for the true positive detection rules (for redacting with). This file will be appended to as new rules are added. + /// public const string DefaultFileName = "RedList.yaml"; /// @@ -27,11 +38,17 @@ public class RowUpdater : OutBase /// The strategy to use to build SQL updates to run on the database /// public IUpdateStrategy UpdateStrategy = new RegexUpdateStrategy(); - + + /// + /// Creates a new instance which stores rules in the (which will also have existing rules loaded from) + /// public RowUpdater(FileInfo rulesFile) : base(rulesFile) { } - + + /// + /// Creates a new instance which stores rules in the + /// public RowUpdater() : this(new FileInfo(DefaultFileName)) { } diff --git a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs index 440e9ad9c..8ca20d339 100644 --- a/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs +++ b/src/applications/IsIdentifiableReviewer/Out/SymbolsRulesFactory.cs @@ -1,15 +1,102 @@ -using System.Text; +using System; +using System.Linq; +using System.Text; using System.Text.RegularExpressions; using Microservices.IsIdentifiable.Reporting; namespace IsIdentifiableReviewer.Out { + /// + /// Determines which bits of a failure get converted to corresponding symbols + /// + public enum SymbolsRuleFactoryMode + { + /// + /// Generates rules that match characters [A-Z]/[a-z] (depending on capitalization of input string) and digits \d + /// + Full, + /// + /// Generates rules that match any digits using \d + /// + DigitsOnly, + + /// + /// Generates rules that match any characters with [A-Z]/[a-z] (depending on capitalization of input string) + /// + CharactersOnly + } + + /// + /// Generates Regex patterns for matching based on permutations of digits (\d) and/or characters([A-Z] or [a-z]). See also . + /// public class SymbolsRulesFactory : IRulePatternFactory { + /// + /// Whether to generate Regex match patterns using the permutation of characters, digits or both. + /// + public SymbolsRuleFactoryMode Mode { get; set; } + + /// + /// Returns just the failing parts expressed as digits and wrapped in capture group(s) e.g. ^(\d\d-\d\d-\d\d).*([A-Z][A-Z]) + /// + /// + /// + /// public string GetPattern(object sender, Failure failure) { StringBuilder sb = new StringBuilder(); + if (failure.HasOverlappingParts(false)) + return FullStringSymbols(sender, failure); + + foreach (var p in failure.Parts.Distinct().OrderBy(p=>p.Offset)) + { + if (p.Offset == 0) + sb.Append("^"); + + //match with capture group the given Word + sb.Append( "("); + + foreach (char cur in p.Word) + { + if (char.IsDigit(cur) && Mode != SymbolsRuleFactoryMode.CharactersOnly) + sb.Append("\\d"); + else + if (char.IsLetter(cur) && Mode != SymbolsRuleFactoryMode.DigitsOnly) + sb.Append(char.IsUpper(cur) ? "[A-Z]" : "[a-z]"); + else + sb.Append(Regex.Escape(cur.ToString())); + } + + sb.Append(")"); + + if (p.Offset + p.Word.Length == failure.ProblemValue.Length) + sb.Append("$"); + else + sb.Append(".*"); + } + + if(sb.Length == 0) + throw new ArgumentException("Failure had no Parts"); + + + //trim last .* + if (sb.ToString().EndsWith(".*")) + return sb.ToString(0, sb.Length - 2); + + return sb.ToString(); + } + + /// + /// Returns a full symbols match of the entire input string (ProblemValue) + /// + /// + /// + /// + private string FullStringSymbols(object sender, Failure failure) + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < failure.ProblemValue.Length; i++) { char cur = failure.ProblemValue[i]; diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs index 472e19382..9186c8740 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/IUpdateStrategy.cs @@ -5,8 +5,19 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies { + /// + /// Interface for generating SQL statements that perform redactions on a database for a given when applying an + /// public interface IUpdateStrategy { + /// + /// Returns SQL that should be run on to redact the by removing the parts matched in + /// + /// Table on which the SQL should be run + /// Cached primary key knowledge about all tables encountered so far, index with to determine primary keys + /// The cell and value in which a problem value was detected + /// How to redact the + /// IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule); } diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs index bb05414be..a6c4850fc 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/ProblemValuesUpdateStrategy.cs @@ -11,6 +11,14 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies /// public class ProblemValuesUpdateStrategy : UpdateStrategy { + /// + /// Generates 1 UPDATE statement per for redacting the current + /// + /// + /// + /// + /// + /// public override IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule) { diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs index 3a5c82fa5..d5c256244 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/RegexUpdateStrategy.cs @@ -17,6 +17,14 @@ public class RegexUpdateStrategy : UpdateStrategy { ProblemValuesUpdateStrategy _fallback = new ProblemValuesUpdateStrategy(); + /// + /// Returns SQL for updating the to redact the capture groups in . If no capture groups are represented in the then this class falls back on + /// + /// + /// + /// + /// + /// public override IEnumerable GetUpdateSql(DiscoveredTable table, Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule) { if (usingRule == null || string.IsNullOrWhiteSpace(usingRule.IfPattern)) diff --git a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs index 878933e68..34fabc673 100644 --- a/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs +++ b/src/applications/IsIdentifiableReviewer/Out/UpdateStrategies/UpdateStrategy.cs @@ -7,8 +7,19 @@ namespace IsIdentifiableReviewer.Out.UpdateStrategies { + /// + /// Abstract implementation of , generates SQL statements for redacting a database + /// public abstract class UpdateStrategy : IUpdateStrategy { + /// + /// Override to generate one or more SQL statements that will fully redact a given in the + /// + /// + /// + /// + /// + /// public abstract IEnumerable GetUpdateSql(DiscoveredTable table,Dictionary primaryKeys, Failure failure, IsIdentifiableRule usingRule); /// diff --git a/src/applications/IsIdentifiableReviewer/Target.cs b/src/applications/IsIdentifiableReviewer/Target.cs index 2adb72543..6c87a13ca 100644 --- a/src/applications/IsIdentifiableReviewer/Target.cs +++ b/src/applications/IsIdentifiableReviewer/Target.cs @@ -1,19 +1,42 @@ using FAnsi; using FAnsi.Discovery; +using Microservices.IsIdentifiable.Reporting; namespace IsIdentifiableReviewer { + /// + /// The location of a database server for which were detected and redaction may take place + /// public class Target { + /// + /// The user friendly name of this database server + /// public string Name {get;set;} + + /// + /// Connection string for connecting to the server + /// public string ConnectionString { get; set; } + + /// + /// The DBMS type, MySql, Sql Server tec + /// public DatabaseType DatabaseType { get; set; } + /// + /// Returns the + /// + /// public override string ToString() { return Name; } + /// + /// Returns a managed object for the for detecting tables, primary keys, running SQL statements etc + /// + /// public DiscoveredServer Discover() { return new DiscoveredServer(ConnectionString, DatabaseType); diff --git a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs index b7d2a81d9..efd65fd4d 100644 --- a/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs +++ b/src/applications/IsIdentifiableReviewer/UnattendedReviewer.cs @@ -5,6 +5,7 @@ using System.Linq; using IsIdentifiableReviewer.Out; using Microservices.IsIdentifiable.Options; +using Microservices.IsIdentifiable.Reporting; using Microservices.IsIdentifiable.Reporting.Destinations; using Microservices.IsIdentifiable.Reporting.Reports; using Microservices.IsIdentifiable.Rules; @@ -13,6 +14,11 @@ namespace IsIdentifiableReviewer { + /// + /// CLI no user interaction mode for running the reviewer application. In this mode all in a table are + /// run through an existing rules base (for detecting true/false positives) and the database is updated to perform redactions. + /// Any failures not covered by existing rules are routed to + /// public class UnattendedReviewer { private readonly Target _target; @@ -21,15 +27,37 @@ public class UnattendedReviewer private readonly IgnoreRuleGenerator _ignorer; private readonly FileInfo _outputFile; + /// + /// The number of that were redacted in the database. Where there are multiple UPDATE statements run per failure, Updates will only be incremented once. + /// public int Updates = 0; + + /// + /// The number of input that were ignored as false positives based on existing ignore rules + /// public int Ignores = 0; + + /// + /// The number of input that were not covered by any existing rules + /// public int Unresolved = 0; + + /// + /// Total number of processed so far + /// public int Total = 0; private Logger _log; Dictionary _updateRulesUsed = new Dictionary(); Dictionary _ignoreRulesUsed = new Dictionary(); + /// + /// Creates a new instance that will connect to the database server () and perform redactions using the + /// + /// CLI options for the process + /// DBMS to connect to for redacting + /// Rules base for detecting false positives + /// Rules base for redacting true positives public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, IgnoreRuleGenerator ignorer, RowUpdater updater) { _log = LogManager.GetCurrentClassLogger(); @@ -56,6 +84,10 @@ public UnattendedReviewer(IsIdentifiableReviewerOptions opts, Target target, Ign _updater = updater; } + /// + /// Connects to the database and runs all failures through the rules base performing redactions as required + /// + /// public int Run() { //In RulesOnly mode this will be null diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java index a488e9b0f..91fe057ca 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/Program.java @@ -88,6 +88,24 @@ private static CommandLine ParseOptions(String[] args) throws ParseException { .hasArg() .longOpt("modality") .build()); + + options.addOption( + Option + .builder("i") + .type(boolean.class) + .argName("identifiable extraction") + .desc("This is an identifiable extraction") + .longOpt("identifiable-extraction") + .build()); + + options.addOption( + Option + .builder("f") + .type(boolean.class) + .argName("no-filters extraction") + .desc("Extraction with no reject filters. True by default if --identifiable-extraction specified") + .longOpt("no-filters-extraction") + .build()); try { commandLine = commLineParser.parse(options, args); diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java index a90a29d9c..1c722871f 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/execution/ExtractorClHost.java @@ -65,10 +65,15 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionName = path.getFileName().toString().replaceFirst("[.][^.]+$", ""); final String projectID = commandLineOptions.getOptionValue("p"); - final String extractionDir = projectID + "/image-requests/" + extractionName; + final String extractionDir = projectID + "/extractions/" + extractionName; + + final boolean isIdentifiableExtraction = commandLineOptions.hasOption("i"); + final boolean isNoFilterExtraction = commandLineOptions.hasOption("f") || isIdentifiableExtraction; _logger.debug("projectID: " + projectID); _logger.debug("extractionDirectory: " + extractionDir); + _logger.debug("isIdentifiableExtraction: " + isIdentifiableExtraction); + _logger.debug("isNoFilterExtraction: " + isNoFilterExtraction); Path fullExtractionDirectory = Paths.get(extractionRoot.getAbsolutePath().toString(), extractionDir); @@ -85,7 +90,7 @@ public ExtractorClHost(GlobalOptions options, CommandLine commandLineOptions, UU String extractionModality = commandLineOptions.getOptionValue("modality", null); - _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, + _csvHandler = new ExtractMessagesCsvHandler(jobIdentifier, projectID, extractionDir, extractionModality, isIdentifiableExtraction, isNoFilterExtraction, rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestProducerOptions), rabbitMQAdapter.SetupProducer(options.ExtractorClOptions.ExtractionRequestInfoProducerOptions)); } diff --git a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java index f3aa28301..a70b5a968 100644 --- a/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java +++ b/src/applications/com.smi.applications.extractorcli/src/main/java/org/smi/extractorcl/fileUtils/ExtractMessagesCsvHandler.java @@ -29,6 +29,8 @@ public class ExtractMessagesCsvHandler implements CsvHandler { private String _projectID; private String _extractionDir; private String _extractionModality; + private boolean _isIdentifiableExtraction; + private boolean _isNoFilterExtraction; private ExtractionKey _extractionKey; private static final Pattern _chiPattern = Pattern.compile("^\\d{10}$"); private static final Pattern _eupiPattern = Pattern.compile("^([A-Z]|[0-9]){32}$"); @@ -51,7 +53,7 @@ public class ExtractMessagesCsvHandler implements CsvHandler { * ExtractRequestInfo messages */ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String extractionDir, - String extractionModality, IProducerModel extractRequestMessageProducerModel, + String extractionModality, boolean isIdentifiableExtraction, boolean isNoFilterExtraction, IProducerModel extractRequestMessageProducerModel, IProducerModel extractRequestInfoMessageProducerModel) { _extractionJobID = extractionJobID; @@ -60,6 +62,8 @@ public ExtractMessagesCsvHandler(UUID extractionJobID, String projectID, String _extractRequestMessageProducerModel = extractRequestMessageProducerModel; _extractRequestInfoMessageProducerModel = extractRequestInfoMessageProducerModel; _extractionModality = extractionModality; + _isIdentifiableExtraction = isIdentifiableExtraction; + _isNoFilterExtraction = isNoFilterExtraction; // TODO(rkm 2020-01-30) Properly handle parsing of the supported modalities if (_extractionModality != null && (!_extractionModality.equals("CT") && !_extractionModality.equals("MR"))) { @@ -147,6 +151,8 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erm.ExtractionDirectory = _extractionDir; erm.JobSubmittedAt = now; erm.KeyTag = _extractionKey.toString(); + erm.IsIdentifiableExtraction = _isIdentifiableExtraction; + erm.IsNoFilterExtraction = _isNoFilterExtraction; if (_extractionKey == ExtractionKey.StudyInstanceUID) erm.ExtractionModality = _extractionModality; @@ -158,6 +164,9 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I erim.JobSubmittedAt = now; erim.KeyValueCount = _identifierSet.size(); erim.KeyTag = _extractionKey.toString(); + erim.IsIdentifiableExtraction = _isIdentifiableExtraction; + erim.IsNoFilterExtraction = _isNoFilterExtraction; + if (_extractionKey == ExtractionKey.StudyInstanceUID) erim.ExtractionModality = _extractionModality; @@ -166,6 +175,8 @@ public void sendMessages(boolean autoRun, int maxIdentifiersPerMessage) throws I sb.append(" ProjectNumber: " + _projectID + System.lineSeparator()); sb.append(" ExtractionDirectory: " + _extractionDir + System.lineSeparator()); sb.append(" ExtractionKey: " + _extractionKey + System.lineSeparator()); + sb.append(" IsIdentifiableExtraction: " + _isIdentifiableExtraction + System.lineSeparator()); + sb.append(" IsNoFilterExtraction: " + _isNoFilterExtraction + System.lineSeparator()); if (_extractionKey == ExtractionKey.StudyInstanceUID) sb.append(" ExtractionModality: " + _extractionModality + System.lineSeparator()); sb.append(" KeyValueCount: " + _identifierSet.size() + System.lineSeparator()); diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java index 17ab5a178..be41bdf77 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/execution/ExtractImagesTest.java @@ -90,6 +90,8 @@ public void testExtractImagesFromSingleFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -152,6 +154,8 @@ public void testExtractImagesFromMultipleFiles() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, + false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); @@ -214,6 +218,8 @@ public void testMissingFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, + false, null, null); @@ -242,6 +248,8 @@ public void testEmptyFile() throws Exception { "MyProjectID", "MyProjectFolder", null, + false, + false, null, null); diff --git a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java index 8c64c76bc..5e3a09366 100644 --- a/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java +++ b/src/applications/com.smi.applications.extractorcli/src/test/java/org/smi/extractorcl/test/fileUtils/ExtractImagesCsvHandlerTest.java @@ -27,7 +27,7 @@ public void testProcessingSingleFile() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, false, false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -83,7 +83,7 @@ public void testProcessingSingleFileWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -138,7 +138,7 @@ public void testProcessingMultipleFilesWithDuplicateSeries() throws Exception { IProducerModel extractRequestInfoMessageProducerModel = mock(IProducerModel.class); UUID uuid = UUID.randomUUID(); - ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -206,7 +206,7 @@ public void testIdentifierSplit() throws LineProcessingException { UUID extractionUid = UUID.randomUUID(); ExtractMessagesCsvHandler handler = new ExtractMessagesCsvHandler(extractionUid, "MyProjectID", - "MyProjectFolder", null, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); + "MyProjectFolder", null,false, false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "SeriesInstanceUID" }); @@ -251,14 +251,14 @@ public void testModalityRequirement() throws LineProcessingException { boolean thrown = false; try { - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "aaaaa",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); } catch (IllegalArgumentException e) { thrown = true; } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null, + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", null,false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); thrown = false; @@ -269,7 +269,7 @@ public void testModalityRequirement() throws LineProcessingException { } assertTrue(thrown); - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); @@ -283,7 +283,7 @@ public void testModalityRequirement() throws LineProcessingException { // Happy path - handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR", + handler = new ExtractMessagesCsvHandler(uuid, "MyProjectID", "MyProjectFolder", "MR",false,false, extractRequestMessageProducerModel, extractRequestInfoMessageProducerModel); handler.processHeader(new String[] { "StudyInstanceUID" }); handler.processLine(1, new String[] { "s1" }); diff --git a/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs b/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs index bd97f0cf2..b72ad8b01 100644 --- a/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs +++ b/src/common/Smi.Common.MongoDb/MongoClientHelpers.cs @@ -10,7 +10,6 @@ namespace Smi.Common.MongoDB { public static class MongoClientHelpers { - private const string MongoServicePasswordVar = "MONGO_SERVICE_PASSWORD"; private const string AuthDatabase = "admin"; // Always authenticate against the admin database private static readonly ILogger _logger = LogManager.GetCurrentClassLogger(); @@ -37,12 +36,10 @@ public static MongoClient GetMongoClient(MongoDbOptions options, string applicat WriteConcern = new WriteConcern(journal: true) }); - string password = Environment.GetEnvironmentVariable(MongoServicePasswordVar, EnvironmentVariableTarget.Process); + if (string.IsNullOrWhiteSpace(options.Password)) + throw new ApplicationException($"MongoDB password must be set"); - if (string.IsNullOrWhiteSpace(password)) - throw new ApplicationException($"MongoDB password must be set in \"{MongoServicePasswordVar}\""); - - MongoCredential credentials = MongoCredential.CreateCredential(AuthDatabase, options.UserName, password); + MongoCredential credentials = MongoCredential.CreateCredential(AuthDatabase, options.UserName, options.Password); var mongoClientSettings = new MongoClientSettings { diff --git a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj index 73d0d104f..3e0ed68b4 100644 --- a/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj +++ b/src/common/Smi.Common.MongoDb/Smi.Common.MongoDb.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/common/Smi.Common/Events/FatalErrorEventArgs.cs b/src/common/Smi.Common/Events/FatalErrorEventArgs.cs index 231ea6be9..593de7f47 100644 --- a/src/common/Smi.Common/Events/FatalErrorEventArgs.cs +++ b/src/common/Smi.Common/Events/FatalErrorEventArgs.cs @@ -6,7 +6,7 @@ namespace Smi.Common.Events { public class FatalErrorEventArgs : EventArgs { - public string Message { get; set; } + public string Message { get; } public Exception Exception { get; } @@ -21,5 +21,14 @@ public FatalErrorEventArgs(BasicReturnEventArgs ra) Message = string.Format("BasicReturnEventArgs: {0} - {1}. (Exchange: {2}, RoutingKey: {3})", ra.ReplyCode, ra.ReplyText, ra.Exchange, ra.RoutingKey); } + + public override string ToString() + { + return "" + + $"{base.ToString()}, " + + $"Message={Message}, " + + $"Exception={Exception}, " + + ""; + } } } diff --git a/src/common/Smi.Common/Execution/MicroserviceHost.cs b/src/common/Smi.Common/Execution/MicroserviceHost.cs index 98c195483..784a70543 100644 --- a/src/common/Smi.Common/Execution/MicroserviceHost.cs +++ b/src/common/Smi.Common/Execution/MicroserviceHost.cs @@ -1,10 +1,5 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Reflection; using DicomTypeTranslation; -using DicomTypeTranslation.Helpers; using JetBrains.Annotations; using NLog; using RabbitMQ.Client; @@ -13,6 +8,10 @@ using Smi.Common.Messages; using Smi.Common.Messaging; using Smi.Common.Options; +using System; +using System.Diagnostics; +using System.IO; +using System.Reflection; namespace Smi.Common.Execution { @@ -75,10 +74,10 @@ protected MicroserviceHost( if (globals.FileSystemOptions.ForceSmiLogsRoot) { - string smiLogsRoot = Environment.GetEnvironmentVariable("SMI_LOGS_ROOT"); + string smiLogsRoot = globals.LogsRoot; if (string.IsNullOrWhiteSpace(smiLogsRoot) || !Directory.Exists(smiLogsRoot)) - throw new ApplicationException($"Invalid logs root: SMI_LOGS_ROOT={smiLogsRoot}"); + throw new ApplicationException($"Invalid logs root: {smiLogsRoot}"); LogManager.Configuration.Variables["baseFileName"] = $"{smiLogsRoot}/{HostProcessName}/${{cached:cached=true:clearCache=None:inner=${{date:format=yyyy-MM-dd-HH-mm-ss}}}}-${{processid}}"; @@ -205,8 +204,6 @@ public virtual void Stop(string reason) /// public void Fatal(string msg, Exception exception) { - Logger.Fatal(exception, msg); - if (_stopCalled) return; diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs index 724a7db61..8dc6b91da 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractFileMessage.cs @@ -24,7 +24,7 @@ public class ExtractFileMessage : ExtractMessage, IFileReferenceMessage, IEquata [JsonConstructor] - private ExtractFileMessage() { } + public ExtractFileMessage() { } public ExtractFileMessage(ExtractionRequestMessage request) : base(request) { } diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs b/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs deleted file mode 100644 index e016dd727..000000000 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatus.cs +++ /dev/null @@ -1,24 +0,0 @@ - -namespace Smi.Common.Messages.Extraction -{ - // TODO(rkm 2020-03-07) Check what errors CTPAnonymiser can actually spit out here - public enum ExtractFileStatus - { - Unknown = 0, - - /// - /// The file has been anonymised successfully - /// - Anonymised, - - /// - /// The file could not be anonymised but will be retried later - /// - ErrorWillRetry, - - /// - /// The file could not be anonymised and will not be retired - /// - ErrorWontRetry - } -} diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs index dd0dd659e..71e6129e1 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractMessage.cs @@ -1,7 +1,9 @@ +using JetBrains.Annotations; using Newtonsoft.Json; using System; + namespace Smi.Common.Messages.Extraction { /// @@ -21,17 +23,31 @@ public abstract class ExtractMessage : IExtractMessage, IEquatable $"ExtractionJobIdentifier={ExtractionJobIdentifier}, " + - $"ProjectNumber={ProjectNumber}, " + - $"ExtractionDirectory={ExtractionDirectory}, " + - $"JobSubmittedAt={JobSubmittedAt}"; + public override string ToString() => + $"ExtractionJobIdentifier={ExtractionJobIdentifier}, " + + $"ProjectNumber={ProjectNumber}, " + + $"ExtractionDirectory={ExtractionDirectory}, " + + $"JobSubmittedAt={JobSubmittedAt}, " + + $"IsIdentifiableExtraction={IsIdentifiableExtraction}, " + + $"IsNoFilterExtraction={IsNoFilterExtraction}, " + + ""; #region Equality Members @@ -55,11 +76,12 @@ public bool Equals(ExtractMessage other) if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return - ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) && - string.Equals(ProjectNumber, other.ProjectNumber) && - string.Equals(ExtractionDirectory, other.ExtractionDirectory) && - JobSubmittedAt.Equals(other.JobSubmittedAt); + return ExtractionJobIdentifier.Equals(other.ExtractionJobIdentifier) + && string.Equals(ProjectNumber, other.ProjectNumber) + && string.Equals(ExtractionDirectory, other.ExtractionDirectory) + && JobSubmittedAt.Equals(other.JobSubmittedAt) + && IsIdentifiableExtraction == other.IsIdentifiableExtraction + && IsNoFilterExtraction == other.IsNoFilterExtraction; } public override bool Equals(object obj) @@ -75,9 +97,11 @@ public override int GetHashCode() unchecked { int hashCode = ExtractionJobIdentifier.GetHashCode(); - hashCode = (hashCode * 397) ^ (ProjectNumber != null ? ProjectNumber.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (ExtractionDirectory != null ? ExtractionDirectory.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); + hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ JobSubmittedAt.GetHashCode(); + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); return hashCode; } } diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs new file mode 100644 index 000000000..0cbab544d --- /dev/null +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatus.cs @@ -0,0 +1,30 @@ +namespace Smi.Common.Messages.Extraction +{ + public enum ExtractedFileStatus + { + /// + /// Unused placeholder value + /// + None = 0, + + /// + /// The file has been anonymised successfully + /// + Anonymised, + + /// + /// The file could not be anonymised and will not be retired + /// + ErrorWontRetry, + + /// + /// The source file could not be found under the given filesystem root + /// + FileMissing, + + /// + /// The source file was successfully copied to the destination + /// + Copied, + } +} diff --git a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs similarity index 52% rename from src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs rename to src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs index 54f0b4d64..0fd69f155 100644 --- a/src/common/Smi.Common/Messages/Extraction/ExtractFileStatusMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileStatusMessage.cs @@ -1,52 +1,63 @@ - + using System; using Newtonsoft.Json; namespace Smi.Common.Messages.Extraction { /// - /// Status message received from the anonymisation service + /// Status message sent by services which extract files (CTP, FileCopier) /// - public class ExtractFileStatusMessage : ExtractMessage, IFileReferenceMessage, IEquatable + public class ExtractedFileStatusMessage : ExtractMessage, IFileReferenceMessage, IEquatable { /// /// Original file path /// + [JsonProperty(Required = Required.Always)] public string DicomFilePath { get; set; } /// - /// The for this file + /// The for this file /// [JsonProperty(Required = Required.Always)] - public ExtractFileStatus Status { get; set; } + public ExtractedFileStatus Status { get; set; } /// - /// Anonymised file name. Only required if a file has been anonymised + /// Output file path, relative to the extraction directory. Only required if an output file has been produced /// - [JsonProperty(Required = Required.DisallowNull)] - public string AnonymisedFileName { get; set; } + [JsonProperty(Required = Required.AllowNull)] + public string OutputFilePath { get; set; } /// /// Message required if Status is not 0 /// - [JsonProperty(Required = Required.DisallowNull)] + [JsonProperty(Required = Required.AllowNull)] public string StatusMessage { get; set; } [JsonConstructor] - public ExtractFileStatusMessage() { } + public ExtractedFileStatusMessage() { } + + public ExtractedFileStatusMessage(IExtractMessage request) + : base(request) { } + public override string ToString() => + $"{base.ToString()}," + + $"DicomFilePath={DicomFilePath}," + + $"ExtractedFileStatus={Status}," + + $"OutputFilePath={OutputFilePath}," + + $"StatusMessage={StatusMessage}," + + ""; #region Equality Members - public bool Equals(ExtractFileStatusMessage other) + public bool Equals(ExtractedFileStatusMessage other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; return base.Equals(other) && Status == other.Status && - string.Equals(AnonymisedFileName, other.AnonymisedFileName) && + string.Equals(OutputFilePath, other.OutputFilePath) && string.Equals(StatusMessage, other.StatusMessage); } @@ -55,7 +66,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((ExtractFileStatusMessage)obj); + return Equals((ExtractedFileStatusMessage)obj); } public override int GetHashCode() @@ -65,18 +76,18 @@ public override int GetHashCode() int hashCode = base.GetHashCode(); hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (int)Status; - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFilePath != null ? OutputFilePath.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (StatusMessage != null ? StatusMessage.GetHashCode() : 0); return hashCode; } } - public static bool operator ==(ExtractFileStatusMessage left, ExtractFileStatusMessage right) + public static bool operator ==(ExtractedFileStatusMessage left, ExtractedFileStatusMessage right) { return Equals(left, right); } - public static bool operator !=(ExtractFileStatusMessage left, ExtractFileStatusMessage right) + public static bool operator !=(ExtractedFileStatusMessage left, ExtractedFileStatusMessage right) { return !Equals(left, right); } diff --git a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs similarity index 50% rename from src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs rename to src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs index f5515bc71..645e3f61c 100644 --- a/src/common/Smi.Common/Messages/Extraction/IsIdentifiableMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/ExtractedFileVerificationMessage.cs @@ -3,8 +3,7 @@ namespace Smi.Common.Messages.Extraction { - // TODO(rkm 2020-02-04) Rename to AnonVerificationMessage - public class IsIdentifiableMessage : ExtractMessage, IFileReferenceMessage, IEquatable + public class ExtractedFileVerificationMessage : ExtractMessage, IFileReferenceMessage, IEquatable { [JsonProperty(Required = Required.Always)] public bool IsIdentifiable { get; set; } @@ -19,42 +18,28 @@ public class IsIdentifiableMessage : ExtractMessage, IFileReferenceMessage, IEqu public string DicomFilePath { get; set; } /// - /// Anonymised file name. Only required if a file has been anonymised + /// Output file path, relative to the extraction directory. Only required if an output file has been produced /// [JsonProperty(Required = Required.Always)] - public string AnonymisedFileName { get; set; } + public string OutputFilePath { get; set; } [JsonConstructor] - public IsIdentifiableMessage() { } + public ExtractedFileVerificationMessage() { } - public IsIdentifiableMessage(Guid extractionJobIdentifier, string projectNumber, string extractionDirectory, DateTime jobSubmittedAt) - : this() - { - ExtractionJobIdentifier = extractionJobIdentifier; - ProjectNumber = projectNumber; - ExtractionDirectory = extractionDirectory; - JobSubmittedAt = jobSubmittedAt; - } - - /// - /// Creates a new instance copying all values from the given origin message - /// - /// - public IsIdentifiableMessage(ExtractFileStatusMessage request) - : this(request.ExtractionJobIdentifier, request.ProjectNumber, request.ExtractionDirectory, - request.JobSubmittedAt) + public ExtractedFileVerificationMessage(ExtractedFileStatusMessage request) + : base(request) { DicomFilePath = request.DicomFilePath; - AnonymisedFileName = request.AnonymisedFileName; + OutputFilePath = request.OutputFilePath; } #region Equality Members - public bool Equals(IsIdentifiableMessage other) + public bool Equals(ExtractedFileVerificationMessage other) { if (ReferenceEquals(null, other)) return false; if (ReferenceEquals(this, other)) return true; - return base.Equals(other) && IsIdentifiable == other.IsIdentifiable && DicomFilePath == other.DicomFilePath && AnonymisedFileName == other.AnonymisedFileName; + return base.Equals(other) && IsIdentifiable == other.IsIdentifiable && DicomFilePath == other.DicomFilePath && OutputFilePath == other.OutputFilePath; } public override bool Equals(object obj) @@ -62,7 +47,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((IsIdentifiableMessage)obj); + return Equals((ExtractedFileVerificationMessage)obj); } public override int GetHashCode() @@ -72,7 +57,7 @@ public override int GetHashCode() int hashCode = base.GetHashCode(); hashCode = (hashCode * 397) ^ IsIdentifiable.GetHashCode(); hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFilePath != null ? OutputFilePath.GetHashCode() : 0); return hashCode; } } diff --git a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs index 6bf2771ec..f6937b31c 100644 --- a/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs +++ b/src/common/Smi.Common/Messages/Extraction/IExtractMessage.cs @@ -30,5 +30,15 @@ public interface IExtractMessage : IMessage /// DateTime the job was submitted at /// DateTime JobSubmittedAt { get; set; } + + /// + /// True if this is an identifiable extraction (i.e. files should not be anonymised) + /// + bool IsIdentifiableExtraction { get; } + + /// + /// True if this is a "no filters" (i.e. no file rejection filters should be applied) + /// + bool IsNoFilterExtraction { get; } } } diff --git a/src/common/Smi.Common/Messages/Messages.cd b/src/common/Smi.Common/Messages/Messages.cd index 27ad9a7cf..ab9f3ef9c 100644 --- a/src/common/Smi.Common/Messages/Messages.cd +++ b/src/common/Smi.Common/Messages/Messages.cd @@ -29,34 +29,16 @@ - - - - - - - - - AAQAAAgAAAAAAAAAgAAAABAASAQAAIAAAAAACCAAAAA= + AAAAAAAAAAAAAAAAgAAAAAAAQAAAAIAAAAAAACAAAAA= Messages\Extraction\ExtractFileMessage.cs - - - - - - - - - - - gAAAAAAAAIAAAAAAgAAAAAIAAAAAAIAAAAAAAAAAAAA= + AAAAAAAAAIAAAAAFgAAAAAIAAAAAAIAAAAAAAAAAAAA= Messages\Extraction\ExtractionRequestMessage.cs @@ -64,7 +46,7 @@ - AAAAAAAAAACAAAAAwAAAAAAAAAACAIAAAAAQAAAAAiA= + AAAAAAAAAAAAAAAEwAAAAAAAAAACAIAAAAAQAAAAAiA= Messages\AccessionDirectoryMessage.cs @@ -72,7 +54,7 @@ - AgAAAAAAAACAAAQEwAAAAAAAIBAAAIAAABAQACAAIiA= + AgAAAAAAAACAAAQEwAAAAAAAIBAAAIAIABAQACAAIiA= Messages\DicomFileMessage.cs @@ -88,7 +70,7 @@ - SAAAAAAAEAAAAAAEgAAAAAECAAgDAAAAAEEAAAAIAAA= + CAAAEAAAEAIAAAAMgAAAAAACABgDgIAAAEEAAAAIIiA= Messages\MessageHeader.cs @@ -115,7 +97,7 @@ - AACAAAAAAAACAAAAgAAAAAIAAAAAAIAAAAAAAAAAAAI= + AACAAAAAAAACAAAGgAAAAAAAAAAAAIAAAAAAAAAAAAA= Messages\Extraction\ExtractFileCollectionInfoMessage.cs @@ -123,31 +105,23 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAIAgAEAAAAAAAAAAAAA= + AAAAAAAAAAAAAAAEgAAAQAAABIAAAMACAAAAAAAAAiI= Messages\Extraction\ExtractMessage.cs - - - - - - - - - - + + - AAAAAAAEAAAAAAAAgAAAAQAAAAAAAIAAAAAAAKAAAAI= - Messages\Extraction\ExtractFileStatusMessage.cs + AAAAAAAEAAAAAAAEgAAAAAAAAAAAAIAAAAAAAKCAAiA= + Messages\Extraction\ExtractedFileStatusMessage.cs - AAAAAAAAAAAAAAAAAAAAAAAAAIAgAEAAAAAAAAAAAAA= + AAAAAAAAAAAAAAAAAAAAQAAABIAAAEACAAAAAAAAAAI= Messages\Extraction\IExtractMessage.cs @@ -168,7 +142,7 @@ - SAAAAAAAEAAAAAAAAAAAAAAAAAgCAAAAAEAAAAAIAAA= + CAAAEAAAEAAAAAAAAAAAAAAAAAgCgAAAAEAAAAAIAAA= Messages\IMessageHeader.cs diff --git a/src/common/Smi.Common/Messaging/Consumer.cs b/src/common/Smi.Common/Messaging/Consumer.cs index 3ebc4472b..274a6a3da 100644 --- a/src/common/Smi.Common/Messaging/Consumer.cs +++ b/src/common/Smi.Common/Messaging/Consumer.cs @@ -213,6 +213,8 @@ protected void Fatal(string msg, Exception exception) _exiting = true; + Logger.Fatal(exception, msg); + ConsumerFatalHandler onFatal = OnFatal; if (onFatal != null) @@ -221,7 +223,6 @@ protected void Fatal(string msg, Exception exception) } else { - Logger.Fatal(exception, msg); throw new Exception("No handlers when attempting to raise OnFatal for this exception", exception); } } diff --git a/src/common/Smi.Common/MicroservicesLogging.cd b/src/common/Smi.Common/MicroservicesLogging.cd index 1ab42de5d..100c8b65f 100644 --- a/src/common/Smi.Common/MicroservicesLogging.cd +++ b/src/common/Smi.Common/MicroservicesLogging.cd @@ -1,7 +1,7 @@  - + @@ -13,7 +13,7 @@ - + @@ -21,67 +21,60 @@ - AAAAAAAAACIAAAAAAAAAAAAAAAAAAAAAIAAAAAEEgAA= + AAQAAAACBDIAEAAAAAAIAAEAAgAAAAhAIAAACAQEgAI= Execution\MicroserviceHost.cs - - - - - - - - AAAAEAAAAALABAAAAAAAAAAAAEAECAAAAAAAAAAEAEA= - Messaging\Consumer.cs - - - - AAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAABAAAAAA= + AAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAA= Execution\MicroserviceHostBootstrapper.cs - - - - - + + + + AAAAAAAAiALABAAAGAAAAQAAAEAEgABIAAgAAAAEAEA= + Messaging\Consumer.cs + + + + + - BAAAAAAAAAAAAAAAQiAAAAAAIAAAAAAAAAAAAAAAIAA= - ProducerModel.cs + BAAAAEAAAAAEAAAAQiAAABCAIAAAAABAAAAAAAAEAQA= + Messaging\ProducerModel.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAA= + AAAAAAAACAAAAAAAAAAAAQAAAAAEAABAAAAAAAAAAAA= Messaging\IConsumer.cs - SAAAAAAAEAAAAAAAAAAAAAAAAAgCAAAAAEAAAAAIAAA= + CAAAEAAAEAAAAAAAAAAAAAAAAAgCgAAAAEAAAAAIAAA= Messages\IMessageHeader.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= Messages\IMessage.cs - - + + - AAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAA= - IProducerModel.cs + AAAAAEAAAAAAAAAAACAAAAAAAAAAAABAAAAAAAAAAAA= + Messaging\IProducerModel.cs diff --git a/src/common/Smi.Common/Options/ConsumerOptions.cs b/src/common/Smi.Common/Options/ConsumerOptions.cs index 74d317e20..825660632 100644 --- a/src/common/Smi.Common/Options/ConsumerOptions.cs +++ b/src/common/Smi.Common/Options/ConsumerOptions.cs @@ -5,7 +5,7 @@ namespace Smi.Common.Options /// /// Configuration options needed to receive messages from a RabbitMQ queue. /// - public class ConsumerOptions + public class ConsumerOptions : IOptions { /// /// Name of the queue to consume from. diff --git a/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs new file mode 100644 index 000000000..1b74ccf54 --- /dev/null +++ b/src/common/Smi.Common/Options/EnvironmentVariableDecorator.cs @@ -0,0 +1,34 @@ +using System; + +namespace Smi.Common.Options +{ + /// + /// Populates values in based on environment variables + /// + public class EnvironmentVariableDecorator : OptionsDecorator + { + public override GlobalOptions Decorate(GlobalOptions options) + { + ForAll(options, SetMongoPassword); + + var logsRoot = Environment.GetEnvironmentVariable("SMI_LOGS_ROOT"); + + if (!string.IsNullOrWhiteSpace(logsRoot)) + options.LogsRoot = logsRoot; + + return options; + } + + private MongoDbOptions SetMongoPassword(MongoDbOptions opt) + { + //get the environment variables current value + string envVar = Environment.GetEnvironmentVariable("MONGO_SERVICE_PASSWORD"); + + //if there's an env var for it and there are mongodb options being used + if (!string.IsNullOrWhiteSpace(envVar) && opt != null) + opt.Password = envVar; + + return opt; + } + } +} diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index 4feba0881..f04141a1b 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -18,52 +18,14 @@ namespace Smi.Common.Options { - public class GlobalOptions + public interface IOptions { - public static GlobalOptions Load(string environment = "default", string currentDirectory = null) - { - IDeserializer deserializer = new DeserializerBuilder() - .WithObjectFactory(GetGlobalOption) - .IgnoreUnmatchedProperties() - .Build(); - - currentDirectory = currentDirectory ?? Environment.CurrentDirectory; - - // Make sure environment ends with yaml - if (!(environment.EndsWith(".yaml") || environment.EndsWith(".yml"))) - environment += ".yaml"; - - // If the yaml file doesn't exist and the path is relative, try looking in currentDirectory instead - if (!File.Exists(environment) && !Path.IsPathRooted(environment)) - environment = Path.Combine(currentDirectory, environment); - - string text = File.ReadAllText(environment); - - var globals = deserializer.Deserialize(new StringReader(text)); - globals.CurrentDirectory = currentDirectory; - globals.MicroserviceOptions = new MicroserviceOptions(); - - return globals; - } - - public static GlobalOptions Load(CliOptions cliOptions) - { - GlobalOptions globalOptions = Load(cliOptions.YamlFile); - globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); - - return globalOptions; - } - - - private static object GetGlobalOption(Type arg) - { - return arg == typeof(GlobalOptions) ? - new GlobalOptions() : - Activator.CreateInstance(arg); - } - private GlobalOptions() { } + } + public class GlobalOptions : IOptions + { + #region AllOptions /// @@ -83,12 +45,14 @@ private GlobalOptions() { } public CohortPackagerOptions CohortPackagerOptions { get; set; } public DicomReprocessorOptions DicomReprocessorOptions { get; set; } public DicomTagReaderOptions DicomTagReaderOptions { get; set; } + public FileCopierOptions FileCopierOptions { get; set; } public IdentifierMapperOptions IdentifierMapperOptions { get; set; } public MongoDbPopulatorOptions MongoDbPopulatorOptions { get; set; } public ProcessDirectoryOptions ProcessDirectoryOptions { get; set; } public DeadLetterReprocessorOptions DeadLetterReprocessorOptions { get; set; } public IsIdentifiableOptions IsIdentifiableOptions { get; set; } + public string LogsRoot { get; set; } #endregion @@ -129,7 +93,7 @@ public class IsIdentifiableOptions : ConsumerOptions } [UsedImplicitly] - public class MicroserviceOptions + public class MicroserviceOptions : IOptions { public bool TraceLogging { get; set; } = true; @@ -147,7 +111,7 @@ public override string ToString() } [UsedImplicitly] - public class ProcessDirectoryOptions + public class ProcessDirectoryOptions : IOptions { public ProducerOptions AccessionDirectoryProducerOptions { get; set; } @@ -158,7 +122,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDbPopulatorOptions + public class MongoDbPopulatorOptions : IOptions { public ConsumerOptions SeriesQueueConsumerOptions { get; set; } public ConsumerOptions ImageQueueConsumerOptions { get; set; } @@ -232,7 +196,7 @@ public IMappingTableOptions Clone() } } - public interface IMappingTableOptions + public interface IMappingTableOptions : IOptions { string MappingConnectionString { get; } string MappingTableName { get; set; } @@ -286,6 +250,15 @@ public override string ToString() } } + [UsedImplicitly] + public class FileCopierOptions : ConsumerOptions + { + public ProducerOptions CopyStatusProducerOptions { get; set; } + public string NoVerifyRoutingKey { get; set; } + + public override string ToString() => GlobalOptions.GenerateToString(this); + } + public enum TagProcessorMode { Serial, @@ -293,7 +266,7 @@ public enum TagProcessorMode } [UsedImplicitly] - public class DicomReprocessorOptions + public class DicomReprocessorOptions : IOptions { public ProcessingMode ProcessingMode { get; set; } @@ -326,11 +299,11 @@ public enum ProcessingMode } [UsedImplicitly] - public class CohortPackagerOptions + public class CohortPackagerOptions : IOptions { public ConsumerOptions ExtractRequestInfoOptions { get; set; } public ConsumerOptions FileCollectionInfoOptions { get; set; } - public ConsumerOptions AnonFailedOptions { get; set; } + public ConsumerOptions NoVerifyStatusOptions { get; set; } public ConsumerOptions VerificationStatusOptions { get; set; } public uint JobWatcherTimeoutInSeconds { get; set; } public string ReporterType { get; set; } @@ -388,8 +361,16 @@ public string AuditorType /// public List Blacklists { get; set; } + public string ExtractAnonRoutingKey { get; set; } + public string ExtractIdentRoutingKey { get; set; } + public ProducerOptions ExtractFilesProducerOptions { get; set; } public ProducerOptions ExtractFilesInfoProducerOptions { get; set; } + + /// + /// ID(s) of ColumnInfo that contains a list of values which should not have data extracted for them. e.g. opt out. The name of the column referenced must match a column in the extraction table + /// + public List RejectColumnInfos { get; set; } public override string ToString() { @@ -435,7 +416,7 @@ public override string ToString() } [UsedImplicitly] - public class DeadLetterReprocessorOptions + public class DeadLetterReprocessorOptions : IOptions { public ConsumerOptions DeadLetterConsumerOptions { get; set; } @@ -450,7 +431,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDatabases + public class MongoDatabases : IOptions { public MongoDbOptions DicomStoreOptions { get; set; } @@ -465,7 +446,7 @@ public override string ToString() } [UsedImplicitly] - public class MongoDbOptions + public class MongoDbOptions : IOptions { public string HostName { get; set; } = "localhost"; public int Port { get; set; } = 27017; @@ -473,6 +454,9 @@ public class MongoDbOptions /// UserName for authentication. If empty, authentication will be skipped. /// public string UserName { get; set; } + + public string Password {get;set;} + public string DatabaseName { get; set; } public bool AreValid(bool skipAuthentication) @@ -493,7 +477,7 @@ public override string ToString() /// Describes the location of the Microsoft Sql Server RDMP platform databases which keep track of load configurations, available datasets (tables) etc /// [UsedImplicitly] - public class RDMPOptions + public class RDMPOptions : IOptions { public string CatalogueConnectionString { get; set; } public string DataExportConnectionString { get; set; } @@ -518,10 +502,10 @@ public override string ToString() /// Describes the root location of all images, file names should be expressed as relative paths (relative to this root). /// [UsedImplicitly] - public class FileSystemOptions + public class FileSystemOptions : IOptions { /// - /// If set, services will require that the "SMI_LOGS_ROOT" environment variable is set and points to a valid directory. + /// If set, services will require that is set and points to a valid directory. /// This helps to ensure that we log to a central location on the production system. /// public bool ForceSmiLogsRoot { get; set; } = false; @@ -555,7 +539,7 @@ public override string ToString() /// /// Describes the location of the rabbit server for sending messages to /// - public class RabbitOptions + public class RabbitOptions : IOptions { public string RabbitMqHostName { get; set; } public int RabbitMqHostPort { get; set; } diff --git a/src/common/Smi.Common/Options/GlobalOptionsFactory.cs b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs new file mode 100644 index 000000000..e7278243b --- /dev/null +++ b/src/common/Smi.Common/Options/GlobalOptionsFactory.cs @@ -0,0 +1,82 @@ +using JetBrains.Annotations; +using System; +using System.Collections.Generic; +using System.IO; +using YamlDotNet.Serialization; + +namespace Smi.Common.Options +{ + public class GlobalOptionsFactory + { + private readonly List _decorators = new List(); + + /// + /// Create a GlobalOptionsFactory with the given set of s. Adds a single by default if passed a null value. + /// + /// + public GlobalOptionsFactory( + [CanBeNull] ICollection decorators = null + ) + { + if (decorators != null) + _decorators.AddRange(decorators); + else + _decorators.Add(new EnvironmentVariableDecorator()); + } + + public GlobalOptions Load(string environment = "default", string currentDirectory = null) + { + IDeserializer deserializer = new DeserializerBuilder() + .WithObjectFactory(GetGlobalOption) + .IgnoreUnmatchedProperties() + .Build(); + + currentDirectory = currentDirectory ?? Environment.CurrentDirectory; + + // Make sure environment ends with yaml + if (!(environment.EndsWith(".yaml") || environment.EndsWith(".yml"))) + environment += ".yaml"; + + // If the yaml file doesn't exist and the path is relative, try looking in currentDirectory instead + if (!File.Exists(environment) && !Path.IsPathRooted(environment)) + environment = Path.Combine(currentDirectory, environment); + + string text = File.ReadAllText(environment); + + var globals = deserializer.Deserialize(new StringReader(text)); + globals.CurrentDirectory = currentDirectory; + globals.MicroserviceOptions = new MicroserviceOptions(); + + return Decorate(globals); + } + + /// + /// Applies all to + /// + /// + /// + private GlobalOptions Decorate(GlobalOptions globals) + { + foreach (var d in _decorators) + globals = d.Decorate(globals); + + return globals; + } + + public GlobalOptions Load(CliOptions cliOptions) + { + GlobalOptions globalOptions = Load(cliOptions.YamlFile, null); + globalOptions.MicroserviceOptions = new MicroserviceOptions(cliOptions); + + // The above Load call does the decoration - don't do it here. + return globalOptions; + } + + private object GetGlobalOption(Type arg) + { + return arg == typeof(GlobalOptions) ? + new GlobalOptions() : + Activator.CreateInstance(arg); + } + } +} diff --git a/src/common/Smi.Common/Options/IOptionsDecorator.cs b/src/common/Smi.Common/Options/IOptionsDecorator.cs new file mode 100644 index 000000000..5ef1d3941 --- /dev/null +++ b/src/common/Smi.Common/Options/IOptionsDecorator.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Smi.Common.Options +{ + /// + /// For classes that modify e.g. populate passwords from a vault etc + /// + public interface IOptionsDecorator + { + GlobalOptions Decorate(GlobalOptions options); + } +} diff --git a/src/common/Smi.Common/Options/Microservices.Common.Options.cd b/src/common/Smi.Common/Options/Microservices.Common.Options.cd index ae6c320f8..cdd71132b 100644 --- a/src/common/Smi.Common/Options/Microservices.Common.Options.cd +++ b/src/common/Smi.Common/Options/Microservices.Common.Options.cd @@ -1,28 +1,187 @@  - - - - AAAAAAAAAAAAEgAAAAAAQQAAAAAQAAAAAAACAIAAAAA= - Options\MicroservicesOptions.cs - - - - - - + - AAAAAEIAEAAAEAAAgAAAAAAAAAAAAIAAAAgEAIAAAAA= + AAAAAEAAAAAAAAAEAAAAAAAAAAAAAQAAAAgAAAAAIAA= Options\ConsumerOptions.cs + - + - AAAAAAAAEAAAEAAAgAgAAAAAAAAAAIAAAAAAAIAAAAA= + AAAAAAAAAAAAAAAEAAAEAAAAAAAAAIAAAAAAAAAAImA= Options\ProducerOptions.cs + + + + + + ACEAAwgAEgAAQACEQIAAAAAAwAAgAAAAAAAAIAAAABA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAAACAAAAAAAACAAAAAAAgAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + + AACAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAAAAAAAAEQACAAAABAAEAAAAAAACAAAAAAAE= + Options\GlobalOptions.cs + + + + + + + QAAAAAAAAAAMAAAFAAAQCAAAAAAAQBAAQAAAEQAAABA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAACAAAQAkAAAAAAAAAAAAAQAAAAEAAAAAAAI= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAIAEAAAA= + Options\GlobalOptions.cs + + + + + + BAAAAAAAAAAgAAAEAAAAAAAAAAAAAAAAAAAAAAABAAA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAAQAIAAAEAAAAAAAAAAAAIBAAOAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAIACQAACACAAAEEAAAIAAAAEAAAAgAAgAAAAAAQQEA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAEAAAAAAAAAIAABEAEAQIgACIAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAIEAAAAAAAAAAAAAAAAAAIAAAAAAEA= + Options\GlobalOptions.cs + + + + + + + AAAAAAACAAAAAIAEAAAAAAAAAAAAAAAAABAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + + gAAAAAAAAAAAAAAEAAAAAAAAKAAAAAACAQAAAAABAAA= + Options\GlobalOptions.cs + + + + + + + AAAAAAAAAAAAAQAEAAAAAAAAAAAAAAAAAAQAACAAAAA= + Options\GlobalOptions.cs + + + + + + + AAAAAQAEAAAAACAEAAACAQABAAAAAAAAAAAAAAAABAA= + Options\GlobalOptions.cs + + + + + + + AAQAAAUAAAAAAAAEAAAAAAABAAAAAAABAAAIAAIAQAA= + Options\GlobalOptions.cs + + + + + + + QAAAAAAAAAAAAAAAAAAQCAAAAAAAQBAAQAAAEAAAABA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAACAAAAA= + Options\GlobalOptions.cs + + + + + + AAAAAAAAAAAAQAAAAAAAAAAAAAAAIAAAAAAAAAAAABA= + Options\GlobalOptions.cs + + \ No newline at end of file diff --git a/src/common/Smi.Common/Options/OptionsDecorator.cs b/src/common/Smi.Common/Options/OptionsDecorator.cs new file mode 100644 index 000000000..44b00a5c2 --- /dev/null +++ b/src/common/Smi.Common/Options/OptionsDecorator.cs @@ -0,0 +1,35 @@ +using System; +using System.Reflection; + +namespace Smi.Common.Options +{ + public abstract class OptionsDecorator : IOptionsDecorator + { + public abstract GlobalOptions Decorate(GlobalOptions options); + + protected void ForAll(IOptions globals, Func setter) where T : IOptions + { + //for each property on branch + foreach (PropertyInfo p in globals.GetType().GetProperties()) + { + var currentValue = p.GetValue(globals); + + //if it's a T then call the action (note that we check the property Type because we are interested in the property even if it is null + if (p.PropertyType.IsAssignableFrom(typeof(T))) + { + //the delegate changes the value of the property of Type T (or creates a new instance from scratch) + var result = setter((T)currentValue); + + //store the result of the delegate for this property + p.SetValue(globals, result); + } + + //process it's children + if (currentValue is IOptions subOptions) + { + ForAll(subOptions, setter); + } + } + } + } +} diff --git a/src/common/Smi.Common/Options/ProducerOptions.cs b/src/common/Smi.Common/Options/ProducerOptions.cs index fb4dc11ed..51ee78630 100644 --- a/src/common/Smi.Common/Options/ProducerOptions.cs +++ b/src/common/Smi.Common/Options/ProducerOptions.cs @@ -4,7 +4,7 @@ namespace Smi.Common.Options /// /// Configuration options needed to send messages to a RabbitMQ exchange /// - public class ProducerOptions + public class ProducerOptions : IOptions { /// /// Name of the RabbitMQ exchange to send messages to diff --git a/src/common/Smi.Common/Options/RabbitMqConfigOptions.md b/src/common/Smi.Common/Options/RabbitMqConfigOptions.md index 2b49fdedb..a20e6ca40 100644 --- a/src/common/Smi.Common/Options/RabbitMqConfigOptions.md +++ b/src/common/Smi.Common/Options/RabbitMqConfigOptions.md @@ -161,10 +161,10 @@ You can then pass the GlobalOptions instance to your specific host in the bootst ```csharp private static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); Parser.Default.ParseArguments(args, cli); - var options = GlobalOptions.Load(cli.YamlFile); + var options = new GlobalOptionsFactory().Load(cli.YamlFile); var bootstrapper = new MicroserviceHostBootstrapper( () => new DicomTagReaderHost(options)); diff --git a/src/common/Smi.Common/README.md b/src/common/Smi.Common/README.md index ba3f9af97..c3d41971f 100644 --- a/src/common/Smi.Common/README.md +++ b/src/common/Smi.Common/README.md @@ -18,7 +18,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); // ... } @@ -35,7 +35,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); // will use the 'default.yaml' file + var options = new GlobalOptionsFactory().Load(); // will use the 'default.yaml' file var consumerOptions = options.MyHostOptions; // you don't really need this here... @@ -111,7 +111,7 @@ public class Program { public static int Main(string[] args) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); var bootstrapper = new MicroserviceHostBootstrapper( () => new CohortPackagerHost(options)); diff --git a/src/common/Smi.Common/RabbitMQAdapter.cd b/src/common/Smi.Common/RabbitMQAdapter.cd index a2056baf7..3323774e6 100644 --- a/src/common/Smi.Common/RabbitMQAdapter.cd +++ b/src/common/Smi.Common/RabbitMQAdapter.cd @@ -1,59 +1,72 @@  - - - - AAgAAAAIAFCAAAQAIAAAAQAAAAAAAAAABAACAAQEBAA= - RabbitMQAdapter.cs - - - - - - - - - - - - AAAAAAAAAAAAEgAAAAAAQQAAAAAQAAAAAAACAIAAAAA= - Options\MicroservicesOptions.cs - - - - - - AAAAAAAAEAAAEAAAgAgAAAAAAAAAAIAAAAAAAIAAIAA= + AAAAAAAAAAAAAAAEgAAEAAAAAAAAAIAAAAAAAAAAImA= Options\ProducerOptions.cs - AAAAAEIAEAAAEAAAgAAAAAAAAAAAAIAAAAgEAIAAIAA= + AAAAAEAAAAAAAAAEAAAAAAAAAAAAAQAAAAgAAAAAIAA= Options\ConsumerOptions.cs - - + + + + + + RabbitMQAdapter.cs + + + + + + RabbitMQAdapter.cs + + + + + RabbitMQAdapter.cs + + + + + AAAAAQAIRJCAAAAEoACwIQAAIAAAAAAIBEACAgAEAAA= + RabbitMQAdapter.cs + + + + + + + + - AAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAA= - IProducerModel.cs + AAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAgAAAAAA= + Options\CliOptions.cs - + - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAA= + AAAAAAAACAAAAAAAAAAAAQAAAAAEAABAAAAAAAAAAAA= Messaging\IConsumer.cs + + + + AAAAAEAAAAAAAAAAACAAAAAAAAAAAABAAAAAAAAAAAA= + Messaging\IProducerModel.cs + + \ No newline at end of file diff --git a/src/common/Smi.Common/Smi.Common.csproj b/src/common/Smi.Common/Smi.Common.csproj index 0333b6580..be5c1ee46 100644 --- a/src/common/Smi.Common/Smi.Common.csproj +++ b/src/common/Smi.Common/Smi.Common.csproj @@ -21,7 +21,6 @@ - @@ -31,16 +30,21 @@ - + + - - - + + + - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/src/common/Smi_Common_Python/Rabbit.py b/src/common/Smi_Common_Python/Rabbit.py index bc20a61ec..cf04fa85f 100755 --- a/src/common/Smi_Common_Python/Rabbit.py +++ b/src/common/Smi_Common_Python/Rabbit.py @@ -113,7 +113,7 @@ def send_CTP_Start_Message(yaml_dict, input_file, extraction_dir, project_name): """ Sends a Message to exchange given by QueueName requesting that CTP anonymises a DICOM file. NOTE! Sends a message direct to CTP input queue, not to the exchange used by CohortExtractor. """ - queueName = yaml_dict['CTPAnonymiserOptions']['ExtractFileConsumerOptions']['QueueName'] + queueName = yaml_dict['CTPAnonymiserOptions']['AnonFileConsumerOptions']['QueueName'] pika_connection = pika.BlockingConnection(get_pika_connection_parameters(yaml_dict)) pika_model = pika_connection.channel() diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java index 61b9e12b4..6f7581241 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/messages/ExtractMessage.java @@ -32,6 +32,18 @@ public abstract class ExtractMessage implements IMessage { @FieldRequired public String JobSubmittedAt; + /** + * True if this is an identifiable extraction (i.e. files should not be anonymised) + */ + @FieldRequired + public boolean IsIdentifiableExtraction; + + /** + * True if this is a "no filters" (i.e. no file rejection filters should be applied) + */ + @FieldRequired + public boolean IsNoFilterExtraction; + protected ExtractMessage() { } } diff --git a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java index 97e70f056..7adfe313e 100644 --- a/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java +++ b/src/common/com.smi.microservices.common/src/main/java/org/smi/common/options/GlobalOptions.java @@ -129,8 +129,9 @@ public String getExtractRoot() { } public class CTPAnonymiserOptions { - - public ConsumerOptions ExtractFileConsumerOptions; + public String VerifyRoutingKey; + public String NoVerifyRoutingKey; + public ConsumerOptions AnonFileConsumerOptions; public ProducerOptions ExtractFileStatusProducerOptions; } diff --git a/src/common/com.smi.microservices.parent/pom.xml b/src/common/com.smi.microservices.parent/pom.xml index 6c4302e67..fb79ce2b7 100644 --- a/src/common/com.smi.microservices.parent/pom.xml +++ b/src/common/com.smi.microservices.parent/pom.xml @@ -86,7 +86,7 @@ maven-resources-plugin - 3.1.0 + 3.2.0 @@ -154,7 +154,7 @@ com.rabbitmq amqp-client - 5.2.0 + 5.9.0 @@ -200,7 +200,7 @@ org.mockito mockito-core - 3.4.6 + 3.5.10 @@ -224,7 +224,7 @@ com.fasterxml.jackson.core jackson-databind - 2.10.1 + 2.11.2 diff --git a/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd b/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd index 1b48c05d8..21957cc37 100644 --- a/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd +++ b/src/microservices/Microservices.CohortExtractor/CohortExtractor.cd @@ -16,7 +16,7 @@ - + @@ -27,7 +27,7 @@ - AAAAAAAEAAICAAAAAAAAAAAAABAAAAAAAEQAAAAAAAA= + AAAAAAAEAAICAAAAAAAAAAAAABAAAgAAAEQAAAAAAAA= Execution\RequestFulfillers\FromCataloguesExtractionRequestFulfiller.cs @@ -45,7 +45,7 @@ - AAAAAAAICCAAAgAAAAAAAAAAAAAAAAAAABACAAAAAAA= + AAAAAAAICCAAAgAAAAAAAEAAAAAAAAAAIDACAAAAAAA= Execution\CohortExtractorHost.cs @@ -55,7 +55,7 @@ - AAAAAAAIAAKAAAEAAgAAAEAAAAAAAAAAABAAAAAAAAA= + AAAAAAAIAQKAAAEAAAAAAEAAAAAAAAAAABAAAAAAAAA= Messaging\ExtractionRequestQueueConsumer.cs @@ -65,7 +65,7 @@ - + AgAAQAAAAAQCAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAA= Execution\ExtractImageCollection.cs @@ -74,7 +74,7 @@ - AAAAAAAEAEAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA= + AAAAAAAEAAAAAAAAAAAAAAAAAAAQAAAAAAAAAQAAAAA= Execution\ProjectPathResolvers\DefaultProjectPathResolver.cs @@ -136,19 +136,16 @@ - + - AAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA= + AAAAAAAAAAACAAAAAAAAAAAAAAAAAgAAAEAAAAAAAAA= Execution\RequestFulfillers\IExtractionRequestFulfiller.cs - - - - AAAAAAAEAEAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAA= + AAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= Execution\ProjectPathResolvers\IProjectPathResolver.cs diff --git a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs index e01abcf08..ba9d04375 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/CohortExtractorHost.cs @@ -78,7 +78,7 @@ public override void Start() InitializeExtractionSources(repositoryLocator); - Consumer = new ExtractionRequestQueueConsumer(_fulfiller, _auditor, _pathResolver, _fileMessageProducer, fileMessageInfoProducer); + Consumer = new ExtractionRequestQueueConsumer(Globals.CohortExtractorOptions, _fulfiller, _auditor, _pathResolver, _fileMessageProducer, fileMessageInfoProducer); RabbitMqAdapter.StartConsumer(_consumerOptions, Consumer, isSolo: false); } @@ -132,6 +132,10 @@ private void InitializeExtractionSources(IRDMPPlatformRepositoryServiceLocator r if(!string.IsNullOrWhiteSpace(_consumerOptions.RejectorType)) _fulfiller.Rejectors.Add(ObjectFactory.CreateInstance(_consumerOptions.RejectorType,typeof(IRejector).Assembly)); + if(_consumerOptions.RejectColumnInfos != null) + foreach(var id in _consumerOptions.RejectColumnInfos) + _fulfiller.Rejectors.Add(new ColumnInfoValuesRejector(repositoryLocator.CatalogueRepository.GetObjectByID(id))); + if(_consumerOptions.Blacklists != null) foreach (int id in _consumerOptions.Blacklists) { diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs new file mode 100644 index 000000000..db208a4a8 --- /dev/null +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnInfoValuesRejector.cs @@ -0,0 +1,51 @@ +using FAnsi.Discovery; +using MapsDirectlyToDatabaseTable; +using NLog; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.QueryBuilding; +using ReusableLibraryCode.DataAccess; +using System; +using System.Collections.Generic; +using System.Data.Common; + + +namespace Microservices.CohortExtractor.Execution.RequestFulfillers +{ + + public class ColumnInfoValuesRejector : ColumnValuesRejector + { + + public ColumnInfoValuesRejector(ColumnInfo columnInfo) : base(columnInfo.GetRuntimeName(),FetchTable(columnInfo)) + { + + } + + private static HashSet FetchTable(ColumnInfo columnInfo) + { + var logger = LogManager.GetCurrentClassLogger(); + HashSet toReturn = new HashSet(StringComparer.CurrentCultureIgnoreCase); + + var qb = new QueryBuilder(limitationSQL: null, hashingAlgorithm: null); + qb.AddColumn(new ColumnInfoToIColumn(new MemoryRepository(), columnInfo)); + + string sql = qb.SQL; + logger.Info("Running PatientID fetch SQL:" + sql); + + DiscoveredTable server = columnInfo.TableInfo.Discover(DataAccessContext.DataExport); + + using (DbConnection con = server.Database.Server.GetConnection()) + { + con.Open(); + DbCommand cmd = server.GetCommand(sql, con); + DbDataReader reader = cmd.ExecuteReader(); + + while (reader.Read()) + toReturn.Add(reader[0].ToString()); + } + + logger.Info($"Found {toReturn.Count} patients in the reject list"); + + return toReturn; + } + } +} diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs new file mode 100644 index 000000000..07c8afed8 --- /dev/null +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/ColumnValuesRejector.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; + + +namespace Microservices.CohortExtractor.Execution.RequestFulfillers +{ + public class ColumnValuesRejector : IRejector + { + private readonly HashSet _rejectPatients; + private readonly string _columnToCheck; + + public ColumnValuesRejector(string column, HashSet values) + { + _columnToCheck = column; + _rejectPatients = values; + } + + public bool Reject(DbDataReader row, out string reason) + { + string patientId; + + try + { + // The patient ID is null + if (row[_columnToCheck] == DBNull.Value) + { + reason = null; + return false; + } + + patientId = (string)row[_columnToCheck]; + } + catch (IndexOutOfRangeException ex) + { + throw new IndexOutOfRangeException($"An error occurred determining the PatientID of the record(s) being extracted. Expected a column called {_columnToCheck}", ex); + } + + if (_rejectPatients.Contains(patientId)) + { + reason = "Patient was in reject list"; + return true; + } + + reason = null; + return false; + } + + } +} diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs index 41a9cfa20..66461ba70 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FakeFulfiller.cs @@ -7,6 +7,10 @@ namespace Microservices.CohortExtractor.Execution.RequestFulfillers { + /// + /// Fake that automatically finds all UIDs that you ask it to look up and returns + /// a single image name for each requested UID. The filename will be the UID(s) you asked for + /// public class FakeFulfiller : IExtractionRequestFulfiller { protected readonly Logger Logger; @@ -17,8 +21,6 @@ public class FakeFulfiller : IExtractionRequestFulfiller public FakeFulfiller() { Logger = LogManager.GetCurrentClassLogger(); - - Logger.Debug("Faking a filename"); } public IEnumerable GetAllMatchingFiles(ExtractionRequestMessage message, IAuditExtractions auditor) diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs index 9b80a3bed..cf7a83be4 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/FromCataloguesExtractionRequestFulfiller.cs @@ -1,8 +1,8 @@ using Microservices.CohortExtractor.Audit; -using Smi.Common.Messages.Extraction; using NLog; using Rdmp.Core.Curation.Data; +using Smi.Common.Messages.Extraction; using System; using System.Collections.Generic; using System.Linq; @@ -41,7 +41,7 @@ protected QueryToExecuteColumnSet[] FilterCatalogues(ICatalogue[] cataloguesToUs public IEnumerable GetAllMatchingFiles(ExtractionRequestMessage message, IAuditExtractions auditor) { var queries = new List(); - + foreach (var c in GetCataloguesFor(message)) { var query = GetQueryToExecute(c, message); @@ -53,12 +53,12 @@ public IEnumerable GetAllMatchingFiles(ExtractionRequest } } - Logger.Debug("Found " + queries.Count + " Catalogues which support extracting based on '" + message.KeyTag + "'"); + Logger.Debug($"Found {queries.Count} Catalogues which support extracting based on '{message.KeyTag}'"); if (queries.Count == 0) throw new Exception($"Couldn't find any compatible Catalogues to run extraction queries against for query {message}"); - + List rejectorsToUse = message.IsNoFilterExtraction ? new List() : Rejectors; foreach (string valueToLookup in message.ExtractionIdentifiers) { @@ -66,10 +66,10 @@ public IEnumerable GetAllMatchingFiles(ExtractionRequest foreach (QueryToExecute query in queries) { - foreach (var result in query.Execute(valueToLookup,Rejectors)) + foreach (QueryToExecuteResult result in query.Execute(valueToLookup, rejectorsToUse)) { - if(!results.ContainsKey(result.SeriesTagValue)) - results.Add(result.SeriesTagValue,new HashSet()); + if (!results.ContainsKey(result.SeriesTagValue)) + results.Add(result.SeriesTagValue, new HashSet()); results[result.SeriesTagValue].Add(result); } @@ -90,7 +90,7 @@ protected virtual QueryToExecute GetQueryToExecute(QueryToExecuteColumnSet colum { if (!string.IsNullOrWhiteSpace(message.Modality)) { - if(ModalityRoutingRegex == null) + if (ModalityRoutingRegex == null) throw new NotSupportedException("Filtering on Modality requires setting a ModalityRoutingRegex"); var anyModality = message.Modality.Split(',', StringSplitOptions.RemoveEmptyEntries); @@ -104,12 +104,12 @@ protected virtual QueryToExecute GetQueryToExecute(QueryToExecuteColumnSet colum } else { - Logger.Log(LogLevel.Warn,nameof(ModalityRoutingRegex) + " did not match Catalogue name " + columnSet.Catalogue.Name); + Logger.Log(LogLevel.Warn, nameof(ModalityRoutingRegex) + " did not match Catalogue name " + columnSet.Catalogue.Name); } } - - + + return new QueryToExecute(columnSet, message.KeyTag); } diff --git a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs index 3d7cae18e..5a2d89fb3 100644 --- a/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs +++ b/src/microservices/Microservices.CohortExtractor/Execution/RequestFulfillers/QueryToExecuteResult.cs @@ -1,5 +1,6 @@ using System; + namespace Microservices.CohortExtractor.Execution.RequestFulfillers { public class QueryToExecuteResult : IEquatable @@ -20,6 +21,9 @@ public QueryToExecuteResult(string filePathValue, string studyTagValue, string s InstanceTagValue = instanceTagValue; Reject = rejection; RejectReason = rejectionReason; + + if (Reject && string.IsNullOrWhiteSpace(RejectReason)) + throw new ArgumentException("RejectReason must be specified if Reject=true"); } public override string ToString() @@ -41,7 +45,7 @@ public override bool Equals(object obj) if (ReferenceEquals(null, obj)) return false; if (ReferenceEquals(this, obj)) return true; if (obj.GetType() != this.GetType()) return false; - return Equals((QueryToExecuteResult) obj); + return Equals((QueryToExecuteResult)obj); } public override int GetHashCode() diff --git a/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs b/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs index 5809d09c0..8bd11e295 100644 --- a/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs +++ b/src/microservices/Microservices.CohortExtractor/Messaging/ExtractionRequestQueueConsumer.cs @@ -1,18 +1,19 @@ - -using System.ComponentModel; using Microservices.CohortExtractor.Audit; using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.ProjectPathResolvers; using Microservices.CohortExtractor.Execution.RequestFulfillers; -using RabbitMQ.Client.Events; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Messaging; +using Smi.Common.Options; +using System.ComponentModel; namespace Microservices.CohortExtractor.Messaging { public class ExtractionRequestQueueConsumer : Consumer { + private readonly CohortExtractorOptions _options; + private readonly IExtractionRequestFulfiller _fulfiller; private readonly IAuditExtractions _auditor; private readonly IProducerModel _fileMessageProducer; @@ -20,10 +21,13 @@ public class ExtractionRequestQueueConsumer : Consumer private readonly IProjectPathResolver _resolver; - public ExtractionRequestQueueConsumer(IExtractionRequestFulfiller fulfiller, IAuditExtractions auditor, + public ExtractionRequestQueueConsumer( + CohortExtractorOptions options, + IExtractionRequestFulfiller fulfiller, IAuditExtractions auditor, IProjectPathResolver pathResolver, IProducerModel fileMessageProducer, IProducerModel fileMessageInfoProducer) { + _options = options; _fulfiller = fulfiller; _auditor = auditor; _resolver = pathResolver; @@ -44,6 +48,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequ } string extractionDirectory = request.ExtractionDirectory.TrimEnd('/', '\\'); + string extractFileRoutingKey = request.IsIdentifiableExtraction ? _options.ExtractIdentRoutingKey : _options.ExtractAnonRoutingKey; foreach (ExtractImageCollection matchedFiles in _fulfiller.GetAllMatchingFiles(request, _auditor)) { @@ -66,7 +71,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractionRequ Logger.Debug($"DicomFilePath={extractFileMessage.DicomFilePath}, OutputPath={extractFileMessage.OutputPath}"); // Send the extract file message - var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header); + var sentHeader = (MessageHeader)_fileMessageProducer.SendMessage(extractFileMessage, header, extractFileRoutingKey); // Record that we sent it infoMessage.ExtractFileMessagesDispatched.Add(sentHeader, extractFileMessage.OutputPath); diff --git a/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj b/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj index a91e9e7f0..0d8799eb8 100644 --- a/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj +++ b/src/microservices/Microservices.CohortExtractor/Microservices.CohortExtractor.csproj @@ -26,7 +26,7 @@ - + diff --git a/src/microservices/Microservices.CohortExtractor/Program.cs b/src/microservices/Microservices.CohortExtractor/Program.cs index b9080c4e1..3eaa99419 100644 --- a/src/microservices/Microservices.CohortExtractor/Program.cs +++ b/src/microservices/Microservices.CohortExtractor/Program.cs @@ -23,7 +23,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( (a) => { - GlobalOptions options = GlobalOptions.Load(a); + GlobalOptions options = new GlobalOptionsFactory().Load(a); var bootStrapper = new MicroserviceHostBootstrapper(() => new CohortExtractorHost(options, null, null)); //Use the auditor and request fullfilers specified in the yaml diff --git a/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs b/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs index 231a05543..39948e677 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/CohortPackagerHost.cs @@ -75,7 +75,7 @@ public override void Start() // TODO(rkm 2020-03-02) Once this is transactional, we can have one "master" service which actually does the job checking RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.ExtractRequestInfoOptions, _requestInfoMessageConsumer, isSolo: true); RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.FileCollectionInfoOptions, _fileCollectionMessageConsumer, isSolo: true); - RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.AnonFailedOptions, _anonFailedMessageConsumer, isSolo: true); + RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.NoVerifyStatusOptions, _anonFailedMessageConsumer, isSolo: true); RabbitMqAdapter.StartConsumer(Globals.CohortPackagerOptions.VerificationStatusOptions, _anonVerificationMessageConsumer, isSolo: true); } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs index af72c65e9..e2718f1c2 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobInfo.cs @@ -1,6 +1,6 @@ +using JetBrains.Annotations; using System; using System.Text; -using JetBrains.Annotations; namespace Microservices.CohortPackager.Execution.ExtractJobStorage { @@ -53,6 +53,11 @@ public class ExtractJobInfo : IEquatable /// public ExtractJobStatus JobStatus { get; } + public bool IsIdentifiableExtraction { get; } + + public bool IsNoFilterExtraction { get; } + + public ExtractJobInfo( Guid extractionJobIdentifier, DateTime jobSubmittedAt, @@ -61,7 +66,10 @@ public ExtractJobInfo( [NotNull] string keyTag, uint keyValueCount, [CanBeNull] string extractionModality, - ExtractJobStatus jobStatus) + ExtractJobStatus jobStatus, + bool isIdentifiableExtraction, + bool isNoFilterExtraction + ) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentNullException(nameof(extractionJobIdentifier)); JobSubmittedAt = (jobSubmittedAt != default(DateTime)) ? jobSubmittedAt : throw new ArgumentNullException(nameof(jobSubmittedAt)); @@ -72,6 +80,8 @@ public ExtractJobInfo( if (extractionModality != null) ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); JobStatus = (jobStatus != default(ExtractJobStatus)) ? jobStatus : throw new ArgumentException(nameof(jobStatus)); + IsIdentifiableExtraction = isIdentifiableExtraction; + IsNoFilterExtraction = isNoFilterExtraction; } public override string ToString() @@ -83,6 +93,8 @@ public override string ToString() sb.AppendLine("KeyTag: " + KeyTag); sb.AppendLine("KeyCount: " + KeyValueCount); sb.AppendLine("ExtractionModality: " + ExtractionModality); + sb.AppendLine("IsIdentifiableExtraction: " + IsIdentifiableExtraction); + sb.AppendLine("IsNoFilterExtraction: " + IsNoFilterExtraction); return sb.ToString(); } @@ -99,6 +111,8 @@ public bool Equals(ExtractJobInfo other) KeyTag == other.KeyTag && KeyValueCount == other.KeyValueCount && ExtractionModality == other.ExtractionModality && + IsIdentifiableExtraction == other.IsIdentifiableExtraction && + IsNoFilterExtraction == other.IsNoFilterExtraction && JobStatus == other.JobStatus; } @@ -123,9 +137,11 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ ProjectNumber.GetHashCode(); hashCode = (hashCode * 397) ^ ExtractionDirectory.GetHashCode(); hashCode = (hashCode * 397) ^ KeyTag.GetHashCode(); - hashCode = (hashCode * 397) ^ (int)KeyValueCount; + hashCode = (hashCode * 397) ^ (int) KeyValueCount; hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); - hashCode = (hashCode * 397) ^ (int)JobStatus; + hashCode = (hashCode * 397) ^ (int) JobStatus; + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); return hashCode; } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs index ac498f88e..127c657ae 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/ExtractJobStore.cs @@ -40,22 +40,22 @@ public void PersistMessageToStore(ExtractFileCollectionInfoMessage message, IMes } public void PersistMessageToStore( - [NotNull] ExtractFileStatusMessage message, + [NotNull] ExtractedFileStatusMessage message, [NotNull] IMessageHeader header) { - if (message.Status == ExtractFileStatus.Unknown) - throw new ApplicationException("ExtractFileStatus was unknown"); - if (message.Status == ExtractFileStatus.Anonymised) + if (message.Status == ExtractedFileStatus.None) + throw new ApplicationException("ExtractedFileStatus was None"); + if (message.Status == ExtractedFileStatus.Anonymised) throw new ApplicationException("Received an anonymisation successful message from the failure queue"); PersistMessageToStoreImpl(message, header); } public void PersistMessageToStore( - [NotNull] IsIdentifiableMessage message, + [NotNull] ExtractedFileVerificationMessage message, [NotNull] IMessageHeader header) { - if (string.IsNullOrWhiteSpace(message.AnonymisedFileName)) + if (string.IsNullOrWhiteSpace(message.OutputFilePath)) throw new ApplicationException("Received a verification message without the AnonymisedFileName set"); if (string.IsNullOrWhiteSpace(message.Report)) throw new ApplicationException("Null or empty report data"); @@ -125,10 +125,18 @@ public IEnumerable> GetCompletedJobVerificationFailures(Gu return GetCompletedJobVerificationFailuresImpl(jobId); } + public IEnumerable GetCompletedJobMissingFileList(Guid jobId) + { + if (jobId == default) + throw new ArgumentNullException(nameof(jobId)); + + return GetCompletedJobMissingFileListImpl(jobId); + } + protected abstract void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header); protected abstract void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header); - protected abstract void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header); - protected abstract void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header); + protected abstract void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header); + protected abstract void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header); protected abstract List GetReadyJobsImpl(Guid specificJobId = new Guid()); protected abstract void CompleteJobImpl(Guid jobId); protected abstract void MarkJobFailedImpl(Guid jobId, Exception e); @@ -136,7 +144,6 @@ public IEnumerable> GetCompletedJobVerificationFailures(Gu protected abstract IEnumerable>> GetCompletedJobRejectionsImpl(Guid jobId); protected abstract IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId); protected abstract IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId); - - + protected abstract IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs index a6d8de53a..8854a16ca 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/IExtractJobStore.cs @@ -26,18 +26,18 @@ public interface IExtractJobStore void PersistMessageToStore(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header); /// - /// Serializes a and it's and stores it + /// Serializes a and it's and stores it /// /// /// - void PersistMessageToStore(ExtractFileStatusMessage fileStatusMessage, IMessageHeader header); + void PersistMessageToStore(ExtractedFileStatusMessage fileStatusMessage, IMessageHeader header); /// - /// Serializes a and it's and stores it + /// Serializes a and it's and stores it /// /// /// - void PersistMessageToStore(IsIdentifiableMessage anonVerificationMessage, IMessageHeader header); + void PersistMessageToStore(ExtractedFileVerificationMessage anonVerificationMessage, IMessageHeader header); /// /// Returns a list of all jobs which are ready for final checks @@ -86,5 +86,12 @@ public interface IExtractJobStore /// /// IEnumerable> GetCompletedJobVerificationFailures(Guid jobId); + + /// + /// Returns the full list of files that were requested but could not be produced, and a reason for each + /// + /// + /// + IEnumerable GetCompletedJobMissingFileList(Guid jobId); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs index b0c1574e4..0ae3a8bde 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensions.cs @@ -13,6 +13,9 @@ public static ExtractJobInfo ToExtractJobInfo(this MongoExtractJobDoc mongoExtra mongoExtractJobDoc.KeyTag, mongoExtractJobDoc.KeyCount, mongoExtractJobDoc.ExtractionModality, - mongoExtractJobDoc.JobStatus); + mongoExtractJobDoc.JobStatus, + mongoExtractJobDoc.IsIdentifiableExtraction, + mongoExtractJobDoc.IsNoFilterExtraction + ); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs index 051316527..3d9413fa1 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStore.cs @@ -70,16 +70,18 @@ protected override void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessa .InsertOne(expectedFilesForKey); } - protected override void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header) + protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header) { if (InCompletedJobCollection(message.ExtractionJobIdentifier)) - throw new ApplicationException("Received an ExtractFileStatusMessage for a job that is already completed"); + throw new ApplicationException("Received an ExtractedFileStatusMessage for a job that is already completed"); var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), - message.AnonymisedFileName, + message.DicomFilePath, + message.OutputFilePath, wasAnonymised: false, isIdentifiable: true, + message.Status, statusMessage: message.StatusMessage); _database @@ -87,16 +89,18 @@ protected override void PersistMessageToStoreImpl(ExtractFileStatusMessage messa .InsertOne(newStatus); } - protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header) + protected override void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header) { if (InCompletedJobCollection(message.ExtractionJobIdentifier)) - throw new ApplicationException("Received an IsIdentifiableMessage for a job that is already completed"); - + throw new ApplicationException("Received an ExtractedFileVerificationMessage for a job that is already completed"); + var newStatus = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(message.ExtractionJobIdentifier, header, _dateTimeProvider), - message.AnonymisedFileName, + message.DicomFilePath, + message.OutputFilePath, wasAnonymised: true, isIdentifiable: message.IsIdentifiable, + ExtractedFileStatus.Anonymised, statusMessage: message.Report); _database @@ -322,26 +326,31 @@ protected override IEnumerable>> GetComple protected override IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId) { // NOTE(rkm 2020-03-16) Files which failed anonymisation should have statuses where WasAnonymised=false and IsIdentifiable=true - FilterDefinition filter = Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); filter &= Builders.Filter.Eq(x => x.WasAnonymised, false); filter &= Builders.Filter.Eq(x => x.IsIdentifiable, true); - - IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); - while (cursor.MoveNext()) - foreach (MongoFileStatusDoc doc in cursor.Current) - yield return new Tuple(doc.AnonymisedFileName, doc.StatusMessage); + return CompletedStatusDocsForFilter(filter); } protected override IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId) { - FilterDefinition filter = Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); filter &= Builders.Filter.Eq(x => x.WasAnonymised, true); filter &= Builders.Filter.Eq(x => x.IsIdentifiable, true); + return CompletedStatusDocsForFilter(filter); + } + protected override IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId) + { + FilterDefinition filter = FilterDefinition.Empty; + filter &= Builders.Filter.Eq(x => x.Header.ExtractionJobIdentifier, jobId); + filter &= Builders.Filter.Eq(x => x.ExtractedFileStatus, ExtractedFileStatus.FileMissing); IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); while (cursor.MoveNext()) foreach (MongoFileStatusDoc doc in cursor.Current) - yield return new Tuple(doc.AnonymisedFileName, doc.StatusMessage); + yield return doc.DicomFilePath; } #region Helper Methods @@ -369,6 +378,14 @@ private bool InCompletedJobCollection(Guid extractionJobIdentifier) .SingleOrDefault() != null; } + private IEnumerable> CompletedStatusDocsForFilter(FilterDefinition filter) + { + IAsyncCursor cursor = _completedStatusCollection.FindSync(filter); + while (cursor.MoveNext()) + foreach (MongoFileStatusDoc doc in cursor.Current) + yield return new Tuple(doc.OutputFileName, doc.StatusMessage); + } + #endregion } } \ No newline at end of file diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs index 36a099286..3530ed5fd 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDoc.cs @@ -8,7 +8,6 @@ namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.Objec { public class MongoCompletedExtractJobDoc : MongoExtractJobDoc, IEquatable { - //TODO(rkm 2020-03-09) Check this is generated by the Bson mapping (derived field) [BsonElement("completedAt")] public DateTime CompletedAt { get; set; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs index cee799505..b9b4230d7 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExpectedFilesDoc.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; -using System.Linq; using JetBrains.Annotations; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; +using System; +using System.Collections.Generic; +using System.Linq; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -14,7 +14,7 @@ namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.Objec /// /// MongoDB document model representing a set of files which are expected to be extracted /// - [BsonIgnoreExtraElements] + [BsonIgnoreExtraElements] // NOTE(rkm 2020-08-28) Required for classes which don't contain a field marked with BsonId public class MongoExpectedFilesDoc : IEquatable { [BsonElement("header")] diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs index 7abbaa11d..1b0e7c4a6 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDoc.cs @@ -1,10 +1,10 @@ -using System; using JetBrains.Annotations; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; +using System; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -45,6 +45,12 @@ public class MongoExtractJobDoc [CanBeNull] public string ExtractionModality { get; set; } + [BsonElement("isIdentifiableExtraction")] + public bool IsIdentifiableExtraction { get; set; } + + [BsonElement("IsNoFilterExtraction")] + public bool IsNoFilterExtraction { get; set; } + [BsonElement("failedJobInfo")] [CanBeNull] public MongoFailedJobInfoDoc FailedJobInfoDoc { get; set; } @@ -60,6 +66,8 @@ public MongoExtractJobDoc( [NotNull] string keyTag, uint keyCount, [CanBeNull] string extractionModality, + bool isIdentifiableExtraction, + bool isNoFilterExtraction, [CanBeNull] MongoFailedJobInfoDoc failedJobInfoDoc) { ExtractionJobIdentifier = (extractionJobIdentifier != default(Guid)) ? extractionJobIdentifier : throw new ArgumentException(nameof(extractionJobIdentifier)); @@ -72,9 +80,12 @@ public MongoExtractJobDoc( KeyCount = (keyCount > 0) ? keyCount : throw new ArgumentNullException(nameof(keyCount)); if (extractionModality != null) ExtractionModality = (!string.IsNullOrWhiteSpace(extractionModality)) ? extractionModality : throw new ArgumentNullException(nameof(extractionModality)); + IsIdentifiableExtraction = isIdentifiableExtraction; + IsNoFilterExtraction = isNoFilterExtraction; FailedJobInfoDoc = failedJobInfoDoc; } + /// /// Copy constructor /// @@ -90,6 +101,7 @@ protected MongoExtractJobDoc(MongoExtractJobDoc existing) KeyCount = existing.KeyCount; ExtractionModality = existing.ExtractionModality; FailedJobInfoDoc = existing.FailedJobInfoDoc; + IsNoFilterExtraction = existing.IsNoFilterExtraction; } public static MongoExtractJobDoc FromMessage( @@ -107,6 +119,8 @@ public static MongoExtractJobDoc FromMessage( message.KeyTag, (uint)message.KeyValueCount, message.ExtractionModality, + message.IsIdentifiableExtraction, + message.IsNoFilterExtraction, null ); } @@ -124,6 +138,8 @@ protected bool Equals(MongoExtractJobDoc other) KeyTag == other.KeyTag && KeyCount == other.KeyCount && ExtractionModality == other.ExtractionModality && + IsIdentifiableExtraction == other.IsIdentifiableExtraction && + IsNoFilterExtraction == other.IsNoFilterExtraction && Equals(FailedJobInfoDoc, other.FailedJobInfoDoc); } @@ -152,6 +168,8 @@ public override int GetHashCode() hashCode = (hashCode * 397) ^ KeyTag.GetHashCode(); hashCode = (hashCode * 397) ^ (int)KeyCount; hashCode = (hashCode * 397) ^ (ExtractionModality != null ? ExtractionModality.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ IsIdentifiableExtraction.GetHashCode(); + hashCode = (hashCode * 397) ^ IsNoFilterExtraction.GetHashCode(); hashCode = (hashCode * 397) ^ (FailedJobInfoDoc != null ? FailedJobInfoDoc.GetHashCode() : 0); return hashCode; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs index 69d2d3f77..917b580df 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDoc.cs @@ -1,23 +1,28 @@ -using System; -using JetBrains.Annotations; +using JetBrains.Annotations; +using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; +using Smi.Common.Messages.Extraction; +using System; +using System.Collections.Generic; +using System.ComponentModel; namespace Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel { - /// - /// - /// - [BsonIgnoreExtraElements] - public class MongoFileStatusDoc + [BsonIgnoreExtraElements] // NOTE(rkm 2020-08-28) Required for classes which don't contain a field marked with BsonId + public class MongoFileStatusDoc : ISupportInitialize { [BsonElement("header")] [NotNull] public MongoExtractionMessageHeaderDoc Header { get; set; } - [BsonElement("anonymisedFileName")] + [BsonElement("dicomFilePath")] [NotNull] - public string AnonymisedFileName { get; set; } + public string DicomFilePath { get; set; } + + [BsonElement("outputFileName")] + [CanBeNull] + public string OutputFileName { get; set; } [BsonElement("wasAnonymised")] public bool WasAnonymised { get; set; } @@ -25,22 +30,57 @@ public class MongoFileStatusDoc [BsonElement("isIdentifiable")] public bool IsIdentifiable { get; set; } + [BsonElement("extractedFileStatus")] + [BsonRepresentation(BsonType.String)] + public ExtractedFileStatus ExtractedFileStatus { get; set; } + + /// + /// Should only be null for identifiable extractions where the file was successfully copied. Otherwise will be the failure reason from CTP or the report content from the IsIdentifiable verification + /// [BsonElement("statusMessage")] - [NotNull] // NOTE(rkm 2020-02-27) Will be the failure reason from an ExtractFileStatusMessage, or the report content from an IsIdentifiableMessage + [CanBeNull] public string StatusMessage { get; set; } + /// + /// Used only to handle old-format documents when deserializing + /// + [BsonExtraElements] + [UsedImplicitly] + public IDictionary ExtraElements { get; set; } + + public MongoFileStatusDoc( [NotNull] MongoExtractionMessageHeaderDoc header, - [NotNull] string anonymisedFileName, + [NotNull] string dicomFilePath, + [CanBeNull] string outputFileName, bool wasAnonymised, bool isIdentifiable, - [NotNull] string statusMessage) + ExtractedFileStatus extractedFileStatus, + [CanBeNull] string statusMessage) { Header = header ?? throw new ArgumentNullException(nameof(header)); - AnonymisedFileName = anonymisedFileName; + DicomFilePath = dicomFilePath ?? throw new ArgumentNullException(nameof(dicomFilePath)); + OutputFileName = outputFileName; WasAnonymised = wasAnonymised; IsIdentifiable = isIdentifiable; - StatusMessage = (!string.IsNullOrWhiteSpace(statusMessage)) ? statusMessage : throw new ArgumentNullException(nameof(statusMessage)); + ExtractedFileStatus = (extractedFileStatus != ExtractedFileStatus.None) ? extractedFileStatus : throw new ArgumentException(nameof(extractedFileStatus)); + StatusMessage = statusMessage; + if (!IsIdentifiable && string.IsNullOrWhiteSpace(statusMessage)) + throw new ArgumentNullException(nameof(statusMessage)); + } + + // ^ISupportInitialize + public void BeginInit() { } + + // ^ISupportInitialize + public void EndInit() + { + if (!ExtraElements.ContainsKey("anonymisedFileName")) + return; + + OutputFileName = (string)ExtraElements["anonymisedFileName"]; + DicomFilePath = ""; + ExtractedFileStatus = OutputFileName == null ? ExtractedFileStatus.ErrorWontRetry : ExtractedFileStatus.Anonymised; } #region Equality Methods @@ -48,9 +88,11 @@ public MongoFileStatusDoc( protected bool Equals(MongoFileStatusDoc other) { return Equals(Header, other.Header) && - AnonymisedFileName == other.AnonymisedFileName && + DicomFilePath == other.DicomFilePath && + OutputFileName == other.OutputFileName && WasAnonymised == other.WasAnonymised && IsIdentifiable == other.IsIdentifiable && + ExtractedFileStatus == other.ExtractedFileStatus && StatusMessage == other.StatusMessage; } @@ -71,9 +113,11 @@ public override int GetHashCode() unchecked { int hashCode = (Header.GetHashCode()); - hashCode = (hashCode * 397) ^ (AnonymisedFileName != null ? AnonymisedFileName.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (DicomFilePath != null ? DicomFilePath.GetHashCode() : 0); + hashCode = (hashCode * 397) ^ (OutputFileName != null ? OutputFileName.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (WasAnonymised.GetHashCode()); hashCode = (hashCode * 397) ^ (IsIdentifiable.GetHashCode()); + hashCode = (hashCode * 397) ^ (ExtractedFileStatus.GetHashCode()); hashCode = (hashCode * 397) ^ (StatusMessage.GetHashCode()); return hashCode; } diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs index c7ba49f58..797312d57 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/ExtractJobWatcher.cs @@ -126,6 +126,8 @@ private void DoJobCompletionTasks(ExtractJobInfo jobInfo) _jobStore.MarkJobCompleted(jobId); _reporter.CreateReport(jobId); + _logger.Info($"Report for {jobId} created"); + _notifier.NotifyJobCompleted(jobInfo); } } diff --git a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs index 11944b15b..9ea387d2d 100644 --- a/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs +++ b/src/microservices/Microservices.CohortPackager/Execution/JobProcessing/Reporting/JobReporterBase.cs @@ -38,22 +38,36 @@ public void CreateReport(Guid jobId) streamWriter.WriteLine(line); streamWriter.WriteLine(); - streamWriter.WriteLine("## Verification failures"); - streamWriter.WriteLine(); - WriteJobVerificationFailures(streamWriter, _jobStore.GetCompletedJobVerificationFailures(jobInfo.ExtractionJobIdentifier)); - streamWriter.WriteLine(); + if (jobInfo.IsIdentifiableExtraction) + { + streamWriter.WriteLine("## Missing file list"); + streamWriter.WriteLine(); + WriteJobMissingFileList(streamWriter, _jobStore.GetCompletedJobMissingFileList(jobInfo.ExtractionJobIdentifier)); + streamWriter.WriteLine(); + } + else + { - streamWriter.WriteLine("## Rejected files"); - streamWriter.WriteLine(); - foreach (Tuple> rejection in _jobStore.GetCompletedJobRejections(jobInfo.ExtractionJobIdentifier)) - WriteJobRejections(streamWriter, rejection); - streamWriter.WriteLine(); + streamWriter.WriteLine("## Verification failures"); + streamWriter.WriteLine(); + WriteJobVerificationFailures(streamWriter, + _jobStore.GetCompletedJobVerificationFailures(jobInfo.ExtractionJobIdentifier)); + streamWriter.WriteLine(); - streamWriter.WriteLine("## Anonymisation failures"); - streamWriter.WriteLine(); - foreach ((string expectedAnonFile, string failureReason) in _jobStore.GetCompletedJobAnonymisationFailures(jobInfo.ExtractionJobIdentifier)) - WriteAnonFailure(streamWriter, expectedAnonFile, failureReason); - streamWriter.WriteLine(); + streamWriter.WriteLine("## Rejected files"); + streamWriter.WriteLine(); + foreach (Tuple> rejection in _jobStore.GetCompletedJobRejections( + jobInfo.ExtractionJobIdentifier)) + WriteJobRejections(streamWriter, rejection); + streamWriter.WriteLine(); + + streamWriter.WriteLine("## Anonymisation failures"); + streamWriter.WriteLine(); + foreach ((string expectedAnonFile, string failureReason) in _jobStore + .GetCompletedJobAnonymisationFailures(jobInfo.ExtractionJobIdentifier)) + WriteAnonFailure(streamWriter, expectedAnonFile, failureReason); + streamWriter.WriteLine(); + } streamWriter.WriteLine("--- end of report ---"); @@ -68,7 +82,10 @@ public void CreateReport(Guid jobId) protected abstract void FinishReport(Stream stream); private static IEnumerable JobHeader(ExtractJobInfo jobInfo) - => new[] + { + string identExtraction = jobInfo.IsIdentifiableExtraction ? "Yes" : "No"; + string filteredExtraction = !jobInfo.IsNoFilterExtraction ? "Yes" : "No"; + var header = new List { $"# SMI file extraction report for {jobInfo.ProjectNumber}", "", @@ -78,13 +95,35 @@ private static IEnumerable JobHeader(ExtractJobInfo jobInfo) $"- Extraction tag: {jobInfo.KeyTag}", $"- Extraction modality: {jobInfo.ExtractionModality ?? "Unspecified"}", $"- Requested identifier count: {jobInfo.KeyValueCount}", + $"- Identifiable extraction: {identExtraction}", + $"- Filtered extraction: {filteredExtraction}", "", - "Report contents:", - "- Verification failures", - "- Rejected failures", - "- Anonymisation failures", }; + if (jobInfo.IsIdentifiableExtraction) + { + header.AddRange(new List + { + "Report contents:", + "- Missing file list (files which were selected from an input ID but could not be found)", + }); + } + else + { + header.AddRange(new List + { + "Report contents:", + "- Verification failures", + " - Summary", + " - Full Details", + "- Rejected failures", + "- Anonymisation failures", + }); + } + + return header; + } + private static void WriteJobRejections(TextWriter streamWriter, Tuple> rejection) { (string rejectionKey, Dictionary rejectionItems) = rejection; @@ -132,39 +171,69 @@ private static void WriteJobVerificationFailures(TextWriter streamWriter, IEnume } } - streamWriter.WriteLine("Summary:"); + streamWriter.WriteLine("### Summary"); streamWriter.WriteLine(); var sb = new StringBuilder(); // Write-out the groupings, ordered by descending count, as a summary without the list of associated files - List>>> grouped = groupedFailures.OrderByDescending(x => x.Value.Sum(y => y.Value.Count)).ToList(); + // Ignore the pixel data here since we deal with it separately below + const string pixelData = "PixelData"; + List>>> grouped = groupedFailures + .Where(x => x.Key != pixelData) + .OrderByDescending(x => x.Value.Sum(y => y.Value.Count)) + .ToList(); + foreach ((string tag, Dictionary> failures) in grouped) { - int totalOccurrences = failures.Sum(x => x.Value.Count); - string line = $"- Tag: {tag} ({totalOccurrences} total occurrence(s))"; - streamWriter.WriteLine(line); - sb.AppendLine(line); - foreach ((string problemVal, List relatedFiles) in failures.OrderByDescending(x => x.Value.Count)) - { - line = $" - Value: '{problemVal}' ({relatedFiles.Count} occurrence(s))"; - streamWriter.WriteLine(line); - sb.AppendLine(line); - foreach (string file in relatedFiles) - sb.AppendLine($" - {file}"); - } + WriteVerificationValuesTag(tag, failures, streamWriter, sb); + WriteVerificationValues(failures.OrderByDescending(x => x.Value.Count), streamWriter, sb); + } - streamWriter.WriteLine(); - sb.AppendLine(); + // Now list the pixel data, which we instead order by decreasing length + if (groupedFailures.TryGetValue(pixelData, out Dictionary> pixelFailures)) + { + WriteVerificationValuesTag(pixelData, pixelFailures, streamWriter, sb); + WriteVerificationValues(pixelFailures.OrderByDescending(x => x.Key.Length), streamWriter, sb); } // Now write-out the same, but with the file listing streamWriter.WriteLine(); - streamWriter.WriteLine("Full details:"); + streamWriter.WriteLine("### Full details"); streamWriter.WriteLine(); streamWriter.Write(sb); } + private static void WriteJobMissingFileList(TextWriter streamWriter, IEnumerable missingFiles) + { + foreach (string file in missingFiles) + streamWriter.WriteLine($"- {file}"); + } + + private static void WriteVerificationValuesTag(string tag, Dictionary> failures, TextWriter streamWriter, StringBuilder sb) + { + int totalOccurrences = failures.Sum(x => x.Value.Count); + string line = $"- Tag: {tag} ({totalOccurrences} total occurrence(s))"; + streamWriter.WriteLine(line); + sb.AppendLine(line); + } + + private static void WriteVerificationValues(IEnumerable>> values, TextWriter streamWriter, StringBuilder sb) + { + + foreach ((string problemVal, List relatedFiles) in values) + { + string line = $" - Value: '{problemVal}' ({relatedFiles.Count} occurrence(s))"; + streamWriter.WriteLine(line); + sb.AppendLine(line); + foreach (string file in relatedFiles) + sb.AppendLine($" - {file}"); + } + + streamWriter.WriteLine(); + sb.AppendLine(); + } + protected abstract void ReleaseUnmanagedResources(); public abstract void Dispose(); ~JobReporterBase() => ReleaseUnmanagedResources(); diff --git a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs index f5cbd910d..dfa582d64 100644 --- a/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs +++ b/src/microservices/Microservices.CohortPackager/Messaging/AnonFailedMessageConsumer.cs @@ -8,10 +8,11 @@ namespace Microservices.CohortPackager.Messaging { + // TODO Naming /// - /// Consumer for (s) + /// Consumer for (s) /// - public class AnonFailedMessageConsumer : Consumer + public class AnonFailedMessageConsumer : Consumer { private readonly IExtractJobStore _store; @@ -21,7 +22,7 @@ public AnonFailedMessageConsumer(IExtractJobStore store) _store = store; } - protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileStatusMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileStatusMessage message, ulong tag) { try { @@ -30,7 +31,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileSta catch (ApplicationException e) { // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage - ErrorAndNack(header, tag, "Error while processing ExtractFileStatusMessage", e); + ErrorAndNack(header, tag, "Error while processing ExtractedFileStatusMessage", e); return; } diff --git a/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs b/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs index 1945e13cb..d353cd842 100644 --- a/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs +++ b/src/microservices/Microservices.CohortPackager/Messaging/AnonVerificationMessageConsumer.cs @@ -12,9 +12,9 @@ namespace Microservices.CohortPackager.Messaging { /// - /// Consumer for (s) + /// Consumer for (s) /// - public class AnonVerificationMessageConsumer : Consumer + public class AnonVerificationMessageConsumer : Consumer { private readonly IExtractJobStore _store; @@ -25,7 +25,7 @@ public AnonVerificationMessageConsumer(IExtractJobStore store) } - protected override void ProcessMessageImpl(IMessageHeader header, IsIdentifiableMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileVerificationMessage message, ulong tag) { try { @@ -45,7 +45,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, IsIdentifiable catch (ApplicationException e) { // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage - ErrorAndNack(header, tag, "Error while processing IsIdentifiableMessage", e); + ErrorAndNack(header, tag, "Error while processing ExtractedFileVerificationMessage", e); return; } diff --git a/src/microservices/Microservices.CohortPackager/Program.cs b/src/microservices/Microservices.CohortPackager/Program.cs index 11135e627..50f60f177 100644 --- a/src/microservices/Microservices.CohortPackager/Program.cs +++ b/src/microservices/Microservices.CohortPackager/Program.cs @@ -22,7 +22,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult((cohortPackagerCliOptions) => { - GlobalOptions globalOptions = GlobalOptions.Load(cohortPackagerCliOptions); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load(cohortPackagerCliOptions); if (cohortPackagerCliOptions.ExtractionId != default) return RecreateReport(globalOptions, cohortPackagerCliOptions.ExtractionId); diff --git a/src/microservices/Microservices.DeadLetterReprocessor/Program.cs b/src/microservices/Microservices.DeadLetterReprocessor/Program.cs index f8fcf2adc..98988dcb7 100644 --- a/src/microservices/Microservices.DeadLetterReprocessor/Program.cs +++ b/src/microservices/Microservices.DeadLetterReprocessor/Program.cs @@ -14,7 +14,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args) .MapResult(deadLetterCliOptions => { - GlobalOptions globals = GlobalOptions.Load(deadLetterCliOptions); + GlobalOptions globals = new GlobalOptionsFactory().Load(deadLetterCliOptions); var bootstrapper = new MicroserviceHostBootstrapper(() => new DeadLetterReprocessorHost(globals, deadLetterCliOptions)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd b/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd index 0f7ce2bbe..65364fd8a 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd +++ b/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd @@ -4,10 +4,10 @@ - + - + @@ -41,7 +41,7 @@ - AAEAAAAAAAAAAAAAAAAAAAAAIAQEAAAAEAAAAAAAAAA= + AAEAAAACAAAAAAAAAAAAAAAAIAQEAAAAEAAAAAAAAAA= Execution\DicomFileMessageToDatasetListProvider.cs @@ -58,7 +58,7 @@ - + AAAAAAAAAAAAAABAAAAAAAACIEAAAAAAAAAAgAAAAAA= Execution\NLogThrowerDataLoadEventListener.cs @@ -123,8 +123,8 @@ Execution\Namers\ICanCreateStagingMyself.cs - - + + diff --git a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj index a0b86b30c..7337a98b8 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj +++ b/src/microservices/Microservices.DicomRelationalMapper/Microservices.DicomRelationalMapper.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/microservices/Microservices.DicomRelationalMapper/Program.cs b/src/microservices/Microservices.DicomRelationalMapper/Program.cs index d60729d2c..e06ae8c0c 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Program.cs +++ b/src/microservices/Microservices.DicomRelationalMapper/Program.cs @@ -12,7 +12,7 @@ private static int Main(string[] args) { return Parser.Default.ParseArguments(args).MapResult((o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomRelationalMapperHost(options)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.DicomReprocessor/Program.cs b/src/microservices/Microservices.DicomReprocessor/Program.cs index a9359cf3b..07acd012d 100644 --- a/src/microservices/Microservices.DicomReprocessor/Program.cs +++ b/src/microservices/Microservices.DicomReprocessor/Program.cs @@ -19,7 +19,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args) .MapResult(dicomReprocessorCliOptions => { - GlobalOptions options = GlobalOptions.Load(dicomReprocessorCliOptions); + GlobalOptions options = new GlobalOptionsFactory().Load(dicomReprocessorCliOptions); var bootStrapper = new MicroserviceHostBootstrapper(() => new DicomReprocessorHost(options, dicomReprocessorCliOptions)); return bootStrapper.Main(); diff --git a/src/microservices/Microservices.DicomTagReader/Program.cs b/src/microservices/Microservices.DicomTagReader/Program.cs index 7500349b5..a85f9b95f 100644 --- a/src/microservices/Microservices.DicomTagReader/Program.cs +++ b/src/microservices/Microservices.DicomTagReader/Program.cs @@ -17,7 +17,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( (o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootstrapper = new MicroserviceHostBootstrapper(() => new DicomTagReaderHost(options)); diff --git a/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs new file mode 100644 index 000000000..945387343 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/ExtractionFileCopier.cs @@ -0,0 +1,87 @@ +using JetBrains.Annotations; +using NLog; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using Smi.Common.Options; +using System; +using System.IO.Abstractions; + + +namespace Microservices.FileCopier.Execution +{ + public class ExtractionFileCopier : IFileCopier + { + [NotNull] private readonly FileCopierOptions _options; + + [NotNull] private readonly IProducerModel _copyStatusProducerModel; + + [NotNull] private readonly string _fileSystemRoot; + [NotNull] private readonly IFileSystem _fileSystem; + + [NotNull] private readonly ILogger _logger; + + + public ExtractionFileCopier( + [NotNull] FileCopierOptions options, + [NotNull] IProducerModel copyStatusCopyStatusProducerModel, + [NotNull] string fileSystemRoot, + [CanBeNull] IFileSystem fileSystem = null) + { + _options = options; + _copyStatusProducerModel = copyStatusCopyStatusProducerModel; + _fileSystemRoot = fileSystemRoot; + _fileSystem = fileSystem ?? new FileSystem(); + + if (!_fileSystem.Directory.Exists(_fileSystemRoot)) + throw new ArgumentException($"Cannot find the specified fileSystemRoot: '{_fileSystemRoot}'"); + + _logger = LogManager.GetLogger(GetType().Name); + + } + + public void ProcessMessage( + [NotNull] ExtractFileMessage message, + [NotNull] IMessageHeader header) + { + string fullSrc = _fileSystem.Path.Combine(_fileSystemRoot, message.DicomFilePath); + + ExtractedFileStatusMessage statusMessage; + + if (!_fileSystem.File.Exists(fullSrc)) + { + statusMessage = new ExtractedFileStatusMessage(message) + { + DicomFilePath = message.DicomFilePath, + Status = ExtractedFileStatus.FileMissing, + StatusMessage = $"Could not find '{fullSrc}'" + }; + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); + return; + } + + string fullDest = _fileSystem.Path.Combine(_fileSystemRoot, message.ExtractionDirectory, message.OutputPath); + + if (_fileSystem.File.Exists(fullDest)) + _logger.Warn($"Output file '{fullDest}' already exists. Will overwrite."); + + IDirectoryInfo parent = _fileSystem.Directory.GetParent(fullDest); + if (!parent.Exists) + { + _logger.Debug($"Creating directory '{parent}'"); + parent.Create(); + } + + _logger.Debug($"Copying source file to '{message.OutputPath}'"); + _fileSystem.File.Copy(fullSrc, fullDest, overwrite: true); + + statusMessage = new ExtractedFileStatusMessage(message) + { + DicomFilePath = message.DicomFilePath, + Status = ExtractedFileStatus.Copied, + OutputFilePath = message.OutputPath, + }; + _ = _copyStatusProducerModel.SendMessage(statusMessage, header, _options.NoVerifyRoutingKey); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs new file mode 100644 index 000000000..7315176a6 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/FileCopierHost.cs @@ -0,0 +1,40 @@ +using JetBrains.Annotations; +using Microservices.FileCopier.Messaging; +using Smi.Common.Execution; +using Smi.Common.Messaging; +using Smi.Common.Options; +using System; +using System.IO; +using System.IO.Abstractions; + + +namespace Microservices.FileCopier.Execution +{ + public class FileCopierHost : MicroserviceHost + { + private readonly FileCopyQueueConsumer _consumer; + + public FileCopierHost( + [NotNull] GlobalOptions options, + [CanBeNull]IFileSystem fileSystem = null, + bool loadSmiLogConfig = true + ) + : base( + options, + loadSmiLogConfig: loadSmiLogConfig + ) + { + Logger.Debug("Creating FileCopierHost with FileSystemRoot: " + Globals.FileSystemOptions.FileSystemRoot); + + IProducerModel copyStatusProducerModel = RabbitMqAdapter.SetupProducer(Globals.FileCopierOptions.CopyStatusProducerOptions, isBatch: false); + + var fileCopier = new ExtractionFileCopier(Globals.FileCopierOptions, copyStatusProducerModel, Globals.FileSystemOptions.FileSystemRoot, fileSystem); + _consumer = new FileCopyQueueConsumer(fileCopier); + } + + public override void Start() + { + RabbitMqAdapter.StartConsumer(Globals.FileCopierOptions, _consumer, isSolo: false); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs b/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs new file mode 100644 index 000000000..808ce30b7 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Execution/IFileCopier.cs @@ -0,0 +1,11 @@ +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; + + +namespace Microservices.FileCopier.Execution +{ + public interface IFileCopier + { + void ProcessMessage(ExtractFileMessage message, IMessageHeader header); + } +} diff --git a/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs new file mode 100644 index 000000000..e7677aae3 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Messaging/FileCopyQueueConsumer.cs @@ -0,0 +1,42 @@ +using JetBrains.Annotations; +using Microservices.FileCopier.Execution; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using System; + +namespace Microservices.FileCopier.Messaging +{ + public class FileCopyQueueConsumer : Consumer + { + [NotNull] private readonly IFileCopier _fileCopier; + + public FileCopyQueueConsumer( + [NotNull] IFileCopier fileCopier) + { + _fileCopier = fileCopier; + } + + protected override void ProcessMessageImpl( + [NotNull] IMessageHeader header, + [NotNull] ExtractFileMessage message, + ulong tag) + { + if (!message.IsIdentifiableExtraction) + throw new ArgumentException("Received a message with IsIdentifiableExtraction not set"); + + try + { + _fileCopier.ProcessMessage(message, header); + } + catch (ApplicationException e) + { + // Catch specific exceptions we are aware of, any uncaught will bubble up to the wrapper in ProcessMessage + ErrorAndNack(header, tag, "Error while processing ExtractedFileStatusMessage", e); + return; + } + + Ack(header, tag); + } + } +} diff --git a/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj b/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj new file mode 100644 index 000000000..3c185c987 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Microservices.FileCopier.csproj @@ -0,0 +1,19 @@ + + FileCopier + Exe + netcoreapp3.1 + false + bin\$(Platform)\$(Configuration)\ + x64 + true + 8.0 + full + true + + + + + + + + diff --git a/src/microservices/Microservices.FileCopier/Program.cs b/src/microservices/Microservices.FileCopier/Program.cs new file mode 100644 index 000000000..1f2bddd69 --- /dev/null +++ b/src/microservices/Microservices.FileCopier/Program.cs @@ -0,0 +1,27 @@ +using CommandLine; +using Microservices.FileCopier.Execution; +using Smi.Common.Execution; +using Smi.Common.Options; + +namespace Microservices.FileCopier +{ + internal static class Program + { + /// + /// Program entry point when run from the command line + /// + /// + private static int Main(string[] args) + { + return Parser.Default.ParseArguments(args).MapResult( + (o) => + { + GlobalOptions options = new GlobalOptionsFactory().Load(o); + + var bootstrapper = new MicroserviceHostBootstrapper(() => new FileCopierHost(options)); + return bootstrapper.Main(); + }, + err => 1); + } + } +} diff --git a/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd b/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd index 2054affeb..064a6c26a 100644 --- a/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd +++ b/src/microservices/Microservices.IdentifierMapper/IdentifierMapper.cd @@ -3,7 +3,7 @@ - AAAAAAAAACAAAgAAAAAAAAAAAAAAAAAAAAACAAAAAAA= + ABACAAAAACAAAgAAAAAAQAAAAAAAAAAAIAACAAAAAAA= Execution\IdentifierMapperHost.cs @@ -16,29 +16,74 @@ AAAAAAAAAAAAAAAAAAAAQAAAIAAAAAIAAAAAAAAAAAA= Messaging\IdentifierMapperControlMessageHandler.cs + + + + + + AAAAAAAAAADAAAAAIAAAQQAQAEAAAAAAAAAAAwAAAAA= + Messaging\IdentifierMapperQueueConsumer.cs + + + + + + AAAAAAAACSAAAgAAAAAAAAgAIAAAQAQAAAAACAAgAAA= + Execution\Swappers\ForGuidIdentifierSwapper.cs + - - + + - AAAAAAAACSAAAgAAAAAAAAAAIAAAQAAAAAAACAAgAAA= - Execution\ForGuidIdentifierSwapper.cs + AAAAAAAACSAAAAAAAAAAAAAAICAAAAAAAAAAiAAAAAA= + Execution\Swappers\PreloadTableSwapper.cs + + + + + + AAAAEAAACqAAAAAgAAAAAAAAIAAAAAAAAABACAAAEAA= + Execution\Swappers\RedisSwapper.cs - - + + - AAAAAAAACSAAAAAAAAAAAAAAACAAAAAAAAAAiAAAAAA= - Execution\IdentifierSwapper.cs + AAAAAAAAGSAAAAAAAAAAAAgAAgACAAAAABBACAAAAAA= + Execution\Swappers\SwapIdentifiers.cs - - - + + + + AAAAAAEgCSAAAAAAQAAAAAAAIAAAAAAAAAAACAAAACA= + Execution\Swappers\TableLookupSwapper.cs + + + + + + AAAAAAAECGAQAAAAAAAAAAAAIAAACAAAAABACAAAAAA= + Execution\Swappers\TableLookupWithGuidFallbackSwapper.cs + + + + + + + + + + + + + + - - + + @@ -52,30 +97,11 @@ - - - - - - - - - - - - - EAAAAAAAAADAAgAAAABAQAAAAAAEAAAAAAAAAAAAAAA= - Messaging\IdentifierMapperQueueConsumer.cs - - - - - - - + + - AAAAAAAACCAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAA= - Execution\ISwapIdentifiers.cs + AAAAAAAACCAAAAAAAAAAAAAAAAAAAAAAAABACAAAAAA= + Execution\Swappers\ISwapIdentifiers.cs diff --git a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj index ef0405e27..346ef70a6 100644 --- a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj +++ b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/microservices/Microservices.IdentifierMapper/Program.cs b/src/microservices/Microservices.IdentifierMapper/Program.cs index e030baa0b..01b16a78e 100644 --- a/src/microservices/Microservices.IdentifierMapper/Program.cs +++ b/src/microservices/Microservices.IdentifierMapper/Program.cs @@ -13,7 +13,7 @@ public static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult( cliOptions => { - GlobalOptions options = GlobalOptions.Load(cliOptions); + GlobalOptions options = new GlobalOptionsFactory().Load(cliOptions); var bootstrapper = new MicroserviceHostBootstrapper(() => new IdentifierMapperHost(options)); return bootstrapper.Main(); diff --git a/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd b/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd index fdd9fbbdf..98a0e3fd2 100644 --- a/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd +++ b/src/microservices/Microservices.IsIdentifiable/IsIdentifiable.cd @@ -12,17 +12,10 @@ - - - - AAAAAgAAAAAABAAAAAAAEAAAAAAASAAAAAAAQAAAQAA= - NerEngine.cs - - - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAQAAAAAAgAAAAAABAAAAIAAACAAAAAACAAAAIAI= Runners\DatabaseRunner.cs @@ -30,25 +23,29 @@ - - + + + + + + + - oAAAAAACAACAAwAABCBAAIAAIACoAAAAAAAAAAAAYBI= + IAAAAAAqACDAA0AABBBEwIAAIACoAABAAABAAADEYAI= Runners\IsIdentifiableAbstractRunner.cs - - - + + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAIAAACAAEIEAAAAAAAAAIAAAAABAAABgAAAAAAA= Reporting\Destinations\CsvDestination.cs @@ -74,14 +71,14 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAIAAAEAAAAAgAAAAAAAAAAAAAAAAAAAAABAA= Reporting\Destinations\DatabaseDestination.cs - AFAABAAAgABAABAAAQAACAAAAAAGAIkABACAAAEEACI= + AFAABAAAgAAAABAAAQAACAAAAAQGBIkABACAAAEEACI= Options\IsIdentifiableAbstractOptions.cs @@ -114,29 +111,19 @@ - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AIAAgAgAAASgAAAAggAAAAACIAAIAAABAAAAAAIAKAI= Runners\DicomFileRunner.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAIAAAAAAAAAUAAAAAAEQAAIAEAAQIAIAACIAEAoAI= Runners\IsIdentifiableMongoRunner.cs - - - - AAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAgAA= - Failure\FailurePart.cs - - - - - @@ -151,56 +138,76 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAAEAAAAAAAAAAAEAAAAAAAABAIBQAAAQACAAAAAA= Reporting\Reports\FailureStoreReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAAAAAAAAAAAAAAABAAAAAAAAAAAAAgAwAAAAEAAA= Reporting\Reports\ColumnFailureReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAA= Reporting\Reports\FailingValuesReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAIAiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQACAAAAAA= Reporting\Reports\PixelTextFailureReport.cs - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + BAIBAAAAAAAAAAAAAAAAAAAAAAEIIAAAABQQCAEAAEA= Reporting\Reports\TreeFailureReport.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAEAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAA= Whitelists\CsvWhitelist.cs - + - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAAAAAAIAAAAAAIAAAAAAAAABAAAAAAAAAAAA= Whitelists\DiscoveredColumnWhitelist.cs + + + + AAEAAAAAAAAAAAAAwAQAAAAAAAAAAIAAAAAAAAAAgAA= + Failures\FailurePart.cs + + + + + + + + + + + BAAAgAgAABAAIACAAAAAAAACgAAAQAAAAAAAAAAAAIQ= + Rules\IsIdentifiableRule.cs + + + - + AAIAAAAAAAAAAAAAAABAAAAAAAAAIAAAAAAAAAAAQAA= Reporting\Reports\IFailureReport.cs @@ -209,22 +216,29 @@ - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + AAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAgAAAAAAA= Reporting\Destinations\IReportDestination.cs - + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAA= Whitelists\IWhitelistSource.cs - - + + + + AAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + Rules\ICustomRule.cs + + + + AIAQAAAAAAAAAAAAAAIAgAAAAAAAAQgBAAAACAEAIQA= - Failure\FailureClassification.cs + Failures\FailureClassification.cs diff --git a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj index 498d5597a..9e5436458 100644 --- a/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj +++ b/src/microservices/Microservices.IsIdentifiable/Microservices.IsIdentifiable.csproj @@ -39,10 +39,10 @@ - - + + diff --git a/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs b/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs index 4d8168efc..72a93342c 100644 --- a/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs +++ b/src/microservices/Microservices.IsIdentifiable/Options/IsIdentifiableAbstractOptions.cs @@ -63,6 +63,9 @@ public abstract class IsIdentifiableAbstractOptions [Option(HelpText = "Optional. Directory of additional rules in yaml format.")] public string RulesDirectory { get; set; } + [Option(HelpText = "Optional. Maximum number of answers to cache per column.",Default = 1_000_000)] + public int MaxValidationCacheSize {get;set;} = 1_000_000; + /// /// Returns a short string with no spaces or punctuation that describes the target. This will be used /// for naming output reports e.g. "biochemistry" , "mydir" etc diff --git a/src/microservices/Microservices.IsIdentifiable/Program.cs b/src/microservices/Microservices.IsIdentifiable/Program.cs index 40ca19772..512d262f8 100644 --- a/src/microservices/Microservices.IsIdentifiable/Program.cs +++ b/src/microservices/Microservices.IsIdentifiable/Program.cs @@ -35,7 +35,7 @@ public static int Main(string[] args) //If running as a self contained micro service (getting messages from RabbitMQ) if (args.Length == 1 && string.Equals(args[0], "--service", StringComparison.CurrentCultureIgnoreCase)) { - var options = GlobalOptions.Load(); + var options = new GlobalOptionsFactory().Load(); var bootstrapper = new MicroserviceHostBootstrapper( () => new IsIdentifiableHost(options)); @@ -115,7 +115,7 @@ private static int Run(IsIdentifiableMongoOptions opts) private static int Run(IsIdentifiableServiceOptions opts) { - var options = GlobalOptions.Load(opts.YamlFile); + var options = new GlobalOptionsFactory().Load(opts.YamlFile); var bootstrapper = new MicroserviceHostBootstrapper( () => new IsIdentifiableHost(options)); diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs index faa714e99..04b462ed7 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/DatabaseRunner.cs @@ -49,10 +49,7 @@ public override int Run() var reader = cmd.ExecuteReader(); - // The query can run in parallel, configure using ISIDENTIFIABLE_NUMTHREADS env var - // XXX default is single-threaded because it breaks during NERd otherwise. - int numThreads = int.Parse(Environment.GetEnvironmentVariable("ISIDENTIFIABLE_NUMTHREADS") ?? "1"); - foreach (Reporting.Failure failure in reader.Cast().AsParallel().WithDegreeOfParallelism(numThreads).SelectMany(GetFailuresIfAny)) + foreach (Failure failure in reader.Cast().SelectMany(GetFailuresIfAny)) AddToReports(failure); CloseReports(); diff --git a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs index b56ee4af2..4073e5bd8 100644 --- a/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs +++ b/src/microservices/Microservices.IsIdentifiable/Runners/IsIdentifiableAbstractRunner.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -10,6 +11,7 @@ using Microservices.IsIdentifiable.Reporting.Reports; using Microservices.IsIdentifiable.Rules; using Microservices.IsIdentifiable.Whitelists; +using Microsoft.Extensions.Caching.Memory; using NLog; using YamlDotNet.Serialization; @@ -88,11 +90,32 @@ public abstract class IsIdentifiableAbstractRunner : IDisposable /// public List CustomWhiteListRules { get; set; } = new List(); + /// + /// One cache per field in the data being evaluated, records the recent values passed to and the results to avoid repeated lookups + /// + public ConcurrentDictionary Caches {get;set;} = new ConcurrentDictionary(); + + /// + /// The maximum size of a Cache before we clear it out to prevent running out of RAM + /// + public int MaxValidationCacheSize {get;set;} + + /// + /// Total number of calls to that were returned from the cache + /// + public long ValidateCacheHits {get;set;} + + /// + /// Total number of calls to that were missing from the cache and run directly + /// + public long ValidateCacheMisses {get;set;} + protected IsIdentifiableAbstractRunner(IsIdentifiableAbstractOptions opts) { _opts = opts; _opts.ValidateOptions(); - + MaxValidationCacheSize = opts.MaxValidationCacheSize; + string targetName = _opts.GetTargetName(); if (opts.ColumnReport) @@ -219,14 +242,43 @@ public void LoadRules(string yaml) // ReSharper disable once UnusedMemberInSuper.Global public abstract int Run(); - + /// /// Returns each subsection of which violates validation rules (e.g. the CHI found). /// /// /// /// - protected IEnumerable Validate(string fieldName, string fieldValue) + protected virtual IEnumerable Validate(string fieldName, string fieldValue) + { + // make sure that we have a cache for this column name + var cache = Caches.GetOrAdd(fieldName,(v)=>new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = MaxValidationCacheSize + })); + + //if we have the cached result use it + if(cache.TryGetValue(fieldValue ?? "NULL",out FailurePart[] result)) + { + ValidateCacheHits++; + return result; + } + + ValidateCacheMisses++; + + //otherwise run ValidateImpl and cache the result + return cache.Set(fieldValue?? "NULL", ValidateImpl(fieldName,fieldValue).ToArray(), new MemoryCacheEntryOptions() { + Size=1 + }); + } + + /// + /// Actual implementation of after a cache miss has occurred. This method is only called when a cached answer is not found for the given and pair + /// + /// + /// + /// + protected virtual IEnumerable ValidateImpl(string fieldName, string fieldValue) { if (_skipColumns.Contains(fieldName)) yield break; @@ -401,6 +453,8 @@ public virtual void Dispose() { foreach (var d in CustomRules.OfType()) d.Dispose(); + + _logger?.Info($"ValidateCacheHits:{ValidateCacheHits} Total ValidateCacheMisses:{ValidateCacheMisses}"); } } } diff --git a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs index 5a387d237..c88f282c4 100644 --- a/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs +++ b/src/microservices/Microservices.IsIdentifiable/Service/IsIdentifiableQueueConsumer.cs @@ -11,7 +11,7 @@ namespace Microservices.IsIdentifiable.Service { - public class IsIdentifiableQueueConsumer : Consumer, IDisposable + public class IsIdentifiableQueueConsumer : Consumer, IDisposable { private readonly IProducerModel _producer; private readonly string _fileSystemRoot; @@ -26,7 +26,7 @@ public IsIdentifiableQueueConsumer(IProducerModel producer, string fileSystemRoo _classifier = classifier; } - protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileStatusMessage message, ulong tag) + protected override void ProcessMessageImpl(IMessageHeader header, ExtractedFileStatusMessage message, ulong tag) { bool isClean = true; object resultObject; @@ -34,10 +34,10 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileSta try { // We should only ever receive messages regarding anonymised images - if (message.Status != ExtractFileStatus.Anonymised) + if (message.Status != ExtractedFileStatus.Anonymised) throw new ApplicationException($"Received a message with anonymised status of {message.Status}"); - var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.AnonymisedFileName) ); + var toProcess = new FileInfo( Path.Combine(_extractionRoot, message.ExtractionDirectory, message.OutputFilePath) ); if(!toProcess.Exists) throw new ApplicationException("IsIdentifiable service cannot find file "+toProcess.FullName); @@ -58,7 +58,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, ExtractFileSta return; } - _producer.SendMessage(new IsIdentifiableMessage(message) + _producer.SendMessage(new ExtractedFileVerificationMessage(message) { IsIdentifiable = ! isClean, Report = JsonConvert.SerializeObject(resultObject) diff --git a/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd b/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd index 56f8806f9..5d3f1b38c 100644 --- a/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd +++ b/src/microservices/Microservices.MongoDbPopulator/MongoDBPopulator.cd @@ -1,7 +1,7 @@  - + AAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAIAACAAAAAAA= Execution\MongoDbPopulatorHost.cs diff --git a/src/microservices/Microservices.MongoDbPopulator/Program.cs b/src/microservices/Microservices.MongoDbPopulator/Program.cs index fa57aab9e..93f58320d 100644 --- a/src/microservices/Microservices.MongoDbPopulator/Program.cs +++ b/src/microservices/Microservices.MongoDbPopulator/Program.cs @@ -16,7 +16,7 @@ private static int Main(string[] args) return Parser.Default.ParseArguments(args).MapResult((o) => { - GlobalOptions options = GlobalOptions.Load(o); + GlobalOptions options = new GlobalOptionsFactory().Load(o); var bootStrapper = new MicroserviceHostBootstrapper(() => new MongoDbPopulatorHost(options)); return bootStrapper.Main(); diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/README.md b/src/microservices/com.smi.microservices.ctpanonymiser/README.md index 07b4a3044..efcbd6c7c 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/README.md +++ b/src/microservices/com.smi.microservices.ctpanonymiser/README.md @@ -21,7 +21,7 @@ The anonymiser is installed via Maven as per the other Java apps, so clone the p | Read/Write | Type | Config setting | | ------------- | ------------- |------------- | -| Read| ExtractFileMessage | `CTPAnonymiserOptions.ExtractFileConsumerOptions` | +| Read| ExtractFileMessage | `CTPAnonymiserOptions.AnonFileConsumerOptions` | | Write| ExtractFileStatusMessage|`CTPAnonymiserOptions.ExtractFileStatusProducerOptions`| The ExtractFileMessage contents include the name of a directory of DICOM files to be anonymised. As the files are anonymised, ExtractFileStatusMessage messages are produced indicating success or otherwise. diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java index 61b3eea81..9c1dc1ccc 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/execution/CTPAnonymiserHost.java @@ -23,7 +23,7 @@ public class CTPAnonymiserHost implements IMicroserviceHost { private final CTPAnonymiserConsumer _consumer; private IProducerModel _producer; - private final GlobalOptions _options; +private final GlobalOptions _options; public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws IOException, TimeoutException { @@ -63,6 +63,7 @@ public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws I SmiCtpProcessor anonTool = new DicomAnonymizerToolBuilder().tagAnonScriptFile(anonScriptFile).check(null).buildDat(); _consumer = new CTPAnonymiserConsumer( + _options, _producer, anonTool, fsRoot, @@ -71,7 +72,7 @@ public CTPAnonymiserHost(GlobalOptions options, CommandLine cliOptions) throws I _logger.info("CTPAnonymiserHost created successfully"); // Start the consumer - _rabbitMqAdapter.StartConsumer(_options.CTPAnonymiserOptions.ExtractFileConsumerOptions, _consumer); + _rabbitMqAdapter.StartConsumer(_options.CTPAnonymiserOptions.AnonFileConsumerOptions, _consumer); } public IProducerModel getProducer() { diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java similarity index 83% rename from src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java rename to src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java index 268c46b9f..1a07d3e2d 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractFileStatusMessage.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messages/ExtractedFileStatusMessage.java @@ -3,20 +3,20 @@ import org.smi.common.messageSerialization.JsonDeserializerWithOptions.FieldRequired; import org.smi.common.messages.ExtractMessage; import org.smi.common.messages.IMessage; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; /** * Message indicating the path to an anonymised file */ -public class ExtractFileStatusMessage extends ExtractMessage implements IMessage { +public class ExtractedFileStatusMessage extends ExtractMessage implements IMessage { @FieldRequired public String DicomFilePath; - public String AnonymisedFileName; + public String OutputFilePath; @FieldRequired - public ExtractFileStatus Status; + public ExtractedFileStatus Status; public String StatusMessage; @@ -46,7 +46,7 @@ public void setProjectNumber(String projectNumber) { ProjectNumber = projectNumber; } - public ExtractFileStatusMessage(ExtractFileMessage request) { + public ExtractedFileStatusMessage(ExtractFileMessage request) { ExtractionJobIdentifier = request.ExtractionJobIdentifier; ExtractionDirectory = request.ExtractionDirectory; @@ -65,7 +65,7 @@ public String toString() { sb.append("DicomFilePath: " + DicomFilePath + "\n"); sb.append("ProjectNumber: " + ProjectNumber + "\n"); sb.append("JobSubmittedAt: " + JobSubmittedAt + "\n"); - sb.append("AnonymisedFileName: " + AnonymisedFileName + "\n"); + sb.append("OutputFilePath: " + OutputFilePath + "\n"); sb.append("Status: " + Status + "\n"); sb.append("StatusMessage: " + StatusMessage + "\n"); diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java index c49b46a49..bd138023b 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/messaging/CTPAnonymiserConsumer.java @@ -11,9 +11,10 @@ import org.smi.common.messaging.SmiConsumer; import org.smi.ctpanonymiser.execution.SmiCtpProcessor; import org.smi.ctpanonymiser.messages.ExtractFileMessage; -import org.smi.ctpanonymiser.messages.ExtractFileStatusMessage; +import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; import org.smi.ctpanonymiser.util.CtpAnonymisationStatus; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.common.options.GlobalOptions; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; import java.io.File; import java.io.FileNotFoundException; @@ -23,8 +24,12 @@ public class CTPAnonymiserConsumer extends SmiConsumer { private final static Logger _logger = Logger.getRootLogger(); - private final static String _routingKey_failure = "failure"; - private final static String _routingKey_success = "success"; + + private GlobalOptions _options; + + private String _routingKey_failure; + private String _routingKey_success; + private String _fileSystemRoot; private String _extractFileSystemRoot; @@ -35,9 +40,12 @@ public class CTPAnonymiserConsumer extends SmiConsumer { private boolean _foundAFile = false; - public CTPAnonymiserConsumer(IProducerModel producer, SmiCtpProcessor anonTool, String fileSystemRoot, + public CTPAnonymiserConsumer(GlobalOptions options, IProducerModel producer, SmiCtpProcessor anonTool, String fileSystemRoot, String extractFileSystemRoot) { + _routingKey_failure = options.CTPAnonymiserOptions.NoVerifyRoutingKey; + _routingKey_success = options.CTPAnonymiserOptions.VerifyRoutingKey; + _statusMessageProducer = producer; _anonTool = anonTool; _fileSystemRoot = fileSystemRoot; @@ -64,7 +72,14 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope return; } - ExtractFileStatusMessage statusMessage = new ExtractFileStatusMessage(extractFileMessage); + if (extractFileMessage.IsIdentifiableExtraction) { + // We should only receive these messages if the queue configuration is wrong, so ok just to crash-out + String msg = "Received a message with IsIdentifiableExtraction set"; + _logger.error(msg); + throw new RuntimeException(msg); + } + + ExtractedFileStatusMessage statusMessage = new ExtractedFileStatusMessage(extractFileMessage); // Got the message, now apply the anonymisation @@ -83,7 +98,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope } statusMessage.StatusMessage = msg; - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.FileMissing; _statusMessageProducer.SendMessage(statusMessage, _routingKey_failure, header); @@ -119,7 +134,7 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope _logger.error(msg); statusMessage.StatusMessage = msg; - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.FileMissing; _statusMessageProducer.SendMessage(statusMessage, _routingKey_failure, header); @@ -136,18 +151,18 @@ public void handleDeliveryImpl(String consumerTag, Envelope envelope, BasicPrope if (!tempFile.delete() || tempFile.exists()) _logger.warn("Could not delete temp file " + tempFile.getAbsolutePath()); - String routingKey = _routingKey_failure; + String routingKey; if (status == CtpAnonymisationStatus.Anonymised) { - statusMessage.AnonymisedFileName = extractFileMessage.OutputPath; - statusMessage.Status = ExtractFileStatus.Anonymised; + statusMessage.OutputFilePath = extractFileMessage.OutputPath; + statusMessage.Status = ExtractedFileStatus.Anonymised; routingKey = _routingKey_success; } else { statusMessage.StatusMessage = _anonTool.getLastStatus(); - statusMessage.Status = ExtractFileStatus.ErrorWontRetry; + statusMessage.Status = ExtractedFileStatus.ErrorWontRetry; routingKey = _routingKey_failure; } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java deleted file mode 100644 index c87f05bf1..000000000 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractFileStatus.java +++ /dev/null @@ -1,22 +0,0 @@ - -package org.smi.ctpanonymiser.util; - -public enum ExtractFileStatus { - - Unknown, - - /** - * The file has been anonymised successfully - */ - Anonymised, - - /** - * The file could not be anonymised but will be retried later - */ - ErrorWillRetry, - - /** - * The file could not be anonymised and will not be retired - */ - ErrorWontRetry; -} diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java new file mode 100644 index 000000000..bbe3a6353 --- /dev/null +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/main/java/org/smi/ctpanonymiser/util/ExtractedFileStatus.java @@ -0,0 +1,29 @@ + +package org.smi.ctpanonymiser.util; + +public enum ExtractedFileStatus { + /** + * Unused placeholder value + */ + None, + + /** + * The file has been anonymised successfully + */ + Anonymised, + + /** + * The file could not be anonymised and will not be retired + */ + ErrorWontRetry, + + /** + * The source file could not be found under the given filesystem root + */ + FileMissing, + + /** + * The source file was successfully copied to the destination + */ + Copied, +} diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java index 6c36016f6..ff4acaba1 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/execution/CTPAnonymiserHostTest.java @@ -16,8 +16,8 @@ import org.smi.ctpanonymiser.Program; import org.smi.ctpanonymiser.execution.CTPAnonymiserHost; import org.smi.ctpanonymiser.messages.ExtractFileMessage; -import org.smi.ctpanonymiser.messages.ExtractFileStatusMessage; -import org.smi.ctpanonymiser.util.ExtractFileStatus; +import org.smi.ctpanonymiser.messages.ExtractedFileStatusMessage; +import org.smi.ctpanonymiser.util.ExtractedFileStatus; import java.io.File; import java.nio.file.Paths; @@ -41,7 +41,7 @@ public class CTPAnonymiserHostTest extends TestCase { private ProducerOptions _extractFileProducerOptions; private ConsumerOptions _extractFileStatusConsumerOptions; private IProducerModel _extractFileMessageProducer; - private AnyConsumer _anonFileStatusMessageConsumer; + private AnyConsumer _anonFileStatusMessageConsumer; private ConnectionFactory _factory; private Connection _conn; @@ -67,15 +67,15 @@ protected void setUp() throws Exception { _options.FileSystemOptions.setFileSystemRoot(_fsRoot); _options.FileSystemOptions.setExtractRoot(_extractRoot); - if (!_options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName.startsWith("TEST.")) - _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName = "TEST." - + _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + if (!_options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName.startsWith("TEST.")) + _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName = "TEST." + + _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName; if (!_options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName.startsWith("TEST.")) _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName = "TEST." + _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; - String _consumerQueueName = _options.CTPAnonymiserOptions.ExtractFileConsumerOptions.QueueName; + String _consumerQueueName = _options.CTPAnonymiserOptions.AnonFileConsumerOptions.QueueName; _producerExchangeName = _options.CTPAnonymiserOptions.ExtractFileStatusProducerOptions.ExchangeName; // Set up RMQ @@ -88,7 +88,7 @@ protected void setUp() throws Exception { _extractFileStatusConsumerOptions.AutoAck = false; _extractFileStatusConsumerOptions.QoSPrefetchCount = 1; - _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractFileStatusMessage.class); + _anonFileStatusMessageConsumer = new AnyConsumer<>(ExtractedFileStatusMessage.class); _extractFileProducerOptions = new ProducerOptions(); _extractFileProducerOptions.ExchangeName = _inputExchName; @@ -103,7 +103,7 @@ protected void setUp() throws Exception { _channel.exchangeDeclare(_inputExchName, "direct", true); _channel.queueDeclare(_consumerQueueName, true, false, false, null); - _channel.queueBind(_consumerQueueName, _inputExchName, ""); + _channel.queueBind(_consumerQueueName, _inputExchName, "anon"); System.out.println(String.format("Bound %s -> %s", _inputExchName, _consumerQueueName)); // Setup the output exch. / queue pair @@ -111,8 +111,8 @@ protected void setUp() throws Exception { _channel.exchangeDeclare(_producerExchangeName, "direct", true); _channel.queueDeclare(_outputQueueName, true, false, false, null); _channel.queueBind(_outputQueueName, _producerExchangeName, ""); - _channel.queueBind(_outputQueueName, _producerExchangeName, "success"); - _channel.queueBind(_outputQueueName, _producerExchangeName, "failure"); + _channel.queueBind(_outputQueueName, _producerExchangeName, "verify"); + _channel.queueBind(_outputQueueName, _producerExchangeName, "noverify"); System.out.println(String.format("Bound %s -> %s", _producerExchangeName, _outputQueueName)); _channel.queuePurge(_consumerQueueName); @@ -164,7 +164,7 @@ public void testBasicAnonymise_Success() throws InterruptedException { TimeUnit.MILLISECONDS.sleep(1000); _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + _extractFileMessageProducer.SendMessage(exMessage, "anon", null); _logger.info("Waiting..."); @@ -185,14 +185,14 @@ public void testBasicAnonymise_Success() throws InterruptedException { if (_anonFileStatusMessageConsumer.isMessageValid()) { - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + ExtractedFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); _logger.info("Message received"); _logger.info("\n" + recvd.toString()); - assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.AnonymisedFileName); + assertEquals("FilePaths do not match", exMessage.OutputPath, recvd.OutputFilePath); assertEquals("Project numbers do not match", exMessage.ProjectNumber, recvd.ProjectNumber); - assertEquals(ExtractFileStatus.Anonymised, recvd.Status); + assertEquals(ExtractedFileStatus.Anonymised, recvd.Status); } else { fail("Did not receive message"); } @@ -214,7 +214,7 @@ public void testBasicAnonymise_Failure() throws InterruptedException { exMessage.ProjectNumber = "123-456"; _logger.info("Sending extract file message to " + _extractFileProducerOptions.ExchangeName); - _extractFileMessageProducer.SendMessage(exMessage, "", null); + _extractFileMessageProducer.SendMessage(exMessage, "anon", null); _logger.info("Waiting..."); @@ -235,13 +235,13 @@ public void testBasicAnonymise_Failure() throws InterruptedException { if (_anonFileStatusMessageConsumer.isMessageValid()) { - ExtractFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); + ExtractedFileStatusMessage recvd = _anonFileStatusMessageConsumer.getMessage(); _logger.info("Message received"); _logger.info("\n" + recvd.toString()); - assertEquals("FilePaths do not match", null, recvd.AnonymisedFileName); - assertEquals(ExtractFileStatus.ErrorWontRetry, recvd.Status); + assertEquals("FilePaths do not match", null, recvd.OutputFilePath); + assertEquals(ExtractedFileStatus.FileMissing, recvd.Status); } else { fail("Did not receive message"); } diff --git a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java index ef47b81bc..cf3e4ce62 100644 --- a/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java +++ b/src/microservices/com.smi.microservices.ctpanonymiser/src/test/java/org/smi/ctpanonymiser/test/messages/ExtractFileMessageTest.java @@ -6,6 +6,7 @@ import org.apache.log4j.Logger; import org.smi.common.logging.SmiLogging; +import org.smi.common.options.GlobalOptions; import org.smi.ctpanonymiser.messages.ExtractFileMessage; import org.smi.ctpanonymiser.messaging.CTPAnonymiserConsumer; @@ -44,10 +45,11 @@ protected void tearDown() throws Exception { super.tearDown(); } - public void testSerializeDeserialize() { + public void testSerializeDeserialize() throws Exception { ExtractFileMessage recvdMessage; - CTPAnonymiserConsumer consumer = new CTPAnonymiserConsumer(null, null, _fileSystemRoot, _extractFileSystemRoot); + GlobalOptions options = GlobalOptions.Load(true); + CTPAnonymiserConsumer consumer = new CTPAnonymiserConsumer(options, null, null, _fileSystemRoot, _extractFileSystemRoot); // Get byte array version of message Gson _gson = new Gson(); diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj index 0dc51d917..978359d8f 100644 --- a/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/Applications.DicomDirectoryProcessor.Tests.csproj @@ -13,12 +13,12 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs b/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs index 7c5821e51..211c42a55 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs +++ b/tests/common/Smi.Common.MongoDb.Tests/MongoQueryParserTests.cs @@ -23,7 +23,7 @@ public void OneTimeSetUp() { TestLogger.Setup(); - GlobalOptions globalOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions globalOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _mongoOptions = globalOptions.MongoDatabases.DicomStoreOptions; } diff --git a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj index 098c117c3..0cb9206a6 100644 --- a/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj +++ b/tests/common/Smi.Common.MongoDb.Tests/Smi.Common.MongoDb.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs b/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs index be38a433f..b1b7ae7e6 100644 --- a/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs +++ b/tests/common/Smi.Common.Tests/DeadLetterMessagingTests/DeadLetterTestHelper.cs @@ -34,7 +34,7 @@ public class DeadLetterTestHelper : IDisposable public void SetUpSuite() { TestLogger.Setup(); - GlobalOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var testConnectionFactory = new ConnectionFactory { diff --git a/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs b/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs index 000e75829..7fa66f323 100644 --- a/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs +++ b/tests/common/Smi.Common.Tests/GlobalOptionsExtensions.cs @@ -60,7 +60,7 @@ public static void UseTestValues(this GlobalOptions g, ConnectionFactory rabbit, g.CohortExtractorOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.ExtractRequestInfoOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.FileCollectionInfoOptions.QoSPrefetchCount = 1; - g.CohortPackagerOptions.AnonFailedOptions.QoSPrefetchCount = 1; + g.CohortPackagerOptions.NoVerifyStatusOptions.QoSPrefetchCount = 1; g.CohortPackagerOptions.VerificationStatusOptions.QoSPrefetchCount = 1; g.DicomTagReaderOptions.QoSPrefetchCount = 1; g.IdentifierMapperOptions.QoSPrefetchCount = 1; diff --git a/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs b/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs index 9b8aed6dd..ef1af3fda 100644 --- a/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs +++ b/tests/common/Smi.Common.Tests/HeaderPreservationTest.cs @@ -14,7 +14,7 @@ public class HeaderPreservationTest [Test] public void SendHeader() { - var o = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var o = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumerOptions = new ConsumerOptions(); consumerOptions.QueueName = "TEST.HeaderPreservationTest_Read1"; diff --git a/tests/common/Smi.Common.Tests/MessageEqualityTests.cs b/tests/common/Smi.Common.Tests/MessageEqualityTests.cs index b962fd535..a462e1b3d 100644 --- a/tests/common/Smi.Common.Tests/MessageEqualityTests.cs +++ b/tests/common/Smi.Common.Tests/MessageEqualityTests.cs @@ -1,9 +1,11 @@  using NUnit.Framework; using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; using System; using System.Linq; + namespace Smi.Common.Tests { public class MessageEqualityTests @@ -112,5 +114,36 @@ public void TestEquals_SeriesMessage() Assert.AreEqual(msg1, msg2); Assert.AreEqual(msg1.GetHashCode(), msg2.GetHashCode()); } + + + private class FooExtractMessage : ExtractMessage { } + + [Test] + public void Tests_ExtractMessage_Equality() + { + Guid guid = Guid.NewGuid(); + DateTime dt = DateTime.UtcNow; + + // TODO(rkm 2020-08-26) Swap these object initializers for proper constructors + var msg1 = new FooExtractMessage + { + JobSubmittedAt = dt, + ExtractionJobIdentifier = guid, + ProjectNumber = "1234", + ExtractionDirectory = "foo/bar", + IsIdentifiableExtraction = true, + }; + var msg2 = new FooExtractMessage + { + JobSubmittedAt = dt, + ExtractionJobIdentifier = guid, + ProjectNumber = "1234", + ExtractionDirectory = "foo/bar", + IsIdentifiableExtraction = true, + }; + + Assert.AreEqual(msg1, msg2); + Assert.AreEqual(msg1.GetHashCode(), msg2.GetHashCode()); + } } } diff --git a/tests/common/Smi.Common.Tests/MicroserviceTester.cs b/tests/common/Smi.Common.Tests/MicroserviceTester.cs index 924f0ba65..90e2b748d 100644 --- a/tests/common/Smi.Common.Tests/MicroserviceTester.cs +++ b/tests/common/Smi.Common.Tests/MicroserviceTester.cs @@ -1,6 +1,5 @@  using RabbitMQ.Client; -using Smi.Common; using Smi.Common.Execution; using Smi.Common.Messages; using Smi.Common.Messaging; @@ -8,6 +7,7 @@ using System; using System.Collections.Generic; + namespace Smi.Common.Tests { public class MicroserviceTester : IDisposable @@ -18,7 +18,7 @@ public class MicroserviceTester : IDisposable private readonly List _declaredExchanges = new List(); private readonly List _declaredQueues = new List(); - private readonly ConnectionFactory _factory; + public readonly ConnectionFactory Factory; /// /// When true, will delete any created queues/exchanges when Dispose is called. Can set to false to inspect @@ -41,7 +41,7 @@ public MicroserviceTester(RabbitOptions rabbitOptions, params ConsumerOptions[] _adapter = new RabbitMqAdapter(rabbitOptions.CreateConnectionFactory(), "TestHost"); - _factory = new ConnectionFactory + Factory = new ConnectionFactory { HostName = rabbitOptions.RabbitMqHostName, Port = rabbitOptions.RabbitMqHostPort, @@ -50,7 +50,7 @@ public MicroserviceTester(RabbitOptions rabbitOptions, params ConsumerOptions[] Password = rabbitOptions.RabbitMqPassword }; - using (var con = _factory.CreateConnection()) + using (var con = Factory.CreateConnection()) using (var model = con.CreateModel()) { //get rid of old exchanges @@ -137,14 +137,14 @@ public void SendMessage(ConsumerOptions toConsumer, IMessageHeader header, IMess /// false to create an entirely new Exchange=>Queue (including deleting any existing queue/exchange). False to simply declare the /// queue and bind it to the exchange which is assumed to already exist (this allows you to set up exchange=>multiple queues). If you are setting up multiple queues /// from a single exchange the first call should be isSecondaryBinding = false and all further calls after that for the same exchange should be isSecondaryBinding=true - public void CreateExchange(string exchangeName, ConsumerOptions consumerIfAny, bool isSecondaryBinding = false) + public void CreateExchange(string exchangeName, string queueName = null, bool isSecondaryBinding = false, string routingKey = "") { if (!exchangeName.Contains("TEST.")) exchangeName = exchangeName.Insert(0, "TEST."); - string queueName = consumerIfAny != null ? consumerIfAny.QueueName : exchangeName; + string queueNameToUse = queueName ?? exchangeName.Replace("Exchange", "Queue"); - using (var con = _factory.CreateConnection()) + using (var con = Factory.CreateConnection()) using (var model = con.CreateModel()) { //setup a sender channel for each of the consumers you want to test sending messages to @@ -153,16 +153,16 @@ public void CreateExchange(string exchangeName, ConsumerOptions consumerIfAny, b if (!isSecondaryBinding) model.ExchangeDelete(exchangeName); - model.QueueDelete(queueName); + model.QueueDelete(queueNameToUse); //Create a binding between the exchange and the queue if (!isSecondaryBinding) model.ExchangeDeclare(exchangeName, ExchangeType.Direct, true);//durable seems to be needed because RabbitMQAdapter wants it? - model.QueueDeclare(queueName, true, false, false); //shared with other users - model.QueueBind(queueName, exchangeName, ""); + model.QueueDeclare(queueNameToUse, true, false, false); //shared with other users + model.QueueBind(queueNameToUse, exchangeName, routingKey); - Console.WriteLine("Created Exchange " + exchangeName + "=>" + queueName); + Console.WriteLine("Created Exchange " + exchangeName + "=>" + queueNameToUse); } } @@ -186,7 +186,7 @@ public void Dispose() if (CleanUpAfterTest) { - using (IConnection conn = _factory.CreateConnection()) + using (IConnection conn = Factory.CreateConnection()) using (IModel model = conn.CreateModel()) { _declaredExchanges.ForEach(x => model.ExchangeDelete(x)); diff --git a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs index 88f9c6e79..e3ae591d8 100644 --- a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs +++ b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs @@ -92,13 +92,11 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM string packageDependency = d.Groups[1].Value; string versionDependency = d.Groups[2].Value; - if (packageDependency.Equals(package)) - { - Assert.AreEqual(version, versionDependency, - "Package {0} is version {1} in {2} but version {3} in {4}", package, version, csproj, - versionDependency, nuspec); - found = true; - } + if (!packageDependency.Equals(package)) continue; + Assert.AreEqual(version, versionDependency, + "Package {0} is version {1} in {2} but version {3} in {4}", package, version, csproj, + versionDependency, nuspec); + found = true; } if (!found) @@ -110,27 +108,20 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM //And make sure it appears in the packages.md file - if (packagesMarkdown != null) + if (packagesMarkdown == null) continue; + found = false; + foreach (string line in File.ReadAllLines(packagesMarkdown)) { - found = false; - foreach (string line in File.ReadAllLines(packagesMarkdown)) + if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) { - if (Regex.IsMatch(line, @"[\s[]" + Regex.Escape(package) + @"[\s\]]", RegexOptions.IgnoreCase)) - { - int count = new Regex(Regex.Escape(version)).Matches(line).Count; - - Assert.GreaterOrEqual(count, 2, - "Markdown file {0} did not contain 2 instances of the version {1} for package {2} in {3}", - packagesMarkdown, version, package, csproj); - found = true; - } + found = true; } - - if (!found) - Assert.Fail("Package {0} in {1} is not documented in {2}. Recommended line is:\r\n{3}", package, - csproj, packagesMarkdown, - BuildRecommendedMarkdownLine(package, version)); } + + if (!found) + Assert.Fail("Package {0} in {1} is not documented in {2}. Recommended line is:\r\n{3}", package, + csproj, packagesMarkdown, + BuildRecommendedMarkdownLine(package, version)); } } diff --git a/tests/common/Smi.Common.Tests/OptionsTests.cs b/tests/common/Smi.Common.Tests/OptionsTests.cs index 870379e2b..a3e6de994 100644 --- a/tests/common/Smi.Common.Tests/OptionsTests.cs +++ b/tests/common/Smi.Common.Tests/OptionsTests.cs @@ -1,7 +1,8 @@  -using System; using NUnit.Framework; using Smi.Common.Options; +using System; +using System.Collections.Generic; namespace Smi.Common.Tests { @@ -12,7 +13,7 @@ public class OptionsTests [TestCase("default.yaml")] public void GlobalOptions_Test(string template) { - GlobalOptions globals = GlobalOptions.Load(template, TestContext.CurrentContext.TestDirectory); + GlobalOptions globals = new GlobalOptionsFactory().Load(template, TestContext.CurrentContext.TestDirectory); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RabbitOptions.RabbitMqHostName)); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.FileSystemOptions.FileSystemRoot)); Assert.IsFalse(string.IsNullOrWhiteSpace(globals.RDMPOptions.CatalogueConnectionString)); @@ -46,7 +47,7 @@ public void TestVerifyPopulatedChecks() [Test] public void Test_GlobalOptionsUseTestValues_Nulls() { - GlobalOptions g = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions g = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); Assert.IsNotNull(g.RabbitOptions.RabbitMqHostName); g.UseTestValues(null, null, null, null, null); @@ -56,10 +57,30 @@ public void Test_GlobalOptionsUseTestValues_Nulls() [Test] public void Test_GlobalOptions_FileReadOption_ThrowsException() { - GlobalOptions g = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions g = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); g.DicomTagReaderOptions.FileReadOption = "SkipLargeTags"; Assert.Throws(() => g.DicomTagReaderOptions.GetReadOption()); } + + + private class TestDecorator : OptionsDecorator + { + public override GlobalOptions Decorate(GlobalOptions options) + { + ForAll(options, (o) => new MongoDbOptions { DatabaseName = "FFFFF" }); + return options; + } + } + + [Test] + public void TestDecorators() + { + var factory = new GlobalOptionsFactory(new List { new TestDecorator() }); + var g = factory.Load(); + Assert.AreEqual("FFFFF", g.MongoDatabases.DeadLetterStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF", g.MongoDatabases.DicomStoreOptions.DatabaseName); + Assert.AreEqual("FFFFF", g.MongoDatabases.ExtractionStoreOptions.DatabaseName); + } } } diff --git a/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs b/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs index 1312a7122..d939132a6 100644 --- a/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs +++ b/tests/common/Smi.Common.Tests/RabbitMqAdapterTests.cs @@ -40,7 +40,7 @@ public void OneTimeSetUp() [SetUp] public void SetUp() { - _testOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _testOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _testProducerOptions = new ProducerOptions { @@ -223,7 +223,7 @@ public void Test_Shutdown(Type consumerType) NLog.Config.SimpleConfigurator.ConfigureForTargetLogging(target, LogLevel.Debug); - var o = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var o = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumer = (IConsumer)Activator.CreateInstance(consumerType); diff --git a/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs b/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs index c8744f7e9..977c35354 100644 --- a/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs +++ b/tests/common/Smi.Common.Tests/RequiresRelationalDb.cs @@ -39,10 +39,11 @@ public void ApplyToContext(TestExecutionContext context) if (server.Exists()) return; + string msg = $"Could not connect to {_type} at '{server.Name}' with the provided connection options"; if (!FailIfUnavailable) - Assert.Ignore(_type + " is not running at '" + server.Name + "'"); + Assert.Ignore(msg); else - Assert.Fail(_type + " is not running at '" + server.Name + "'"); + Assert.Fail(msg); } public static ConStrs GetRelationalDatabaseConnectionStrings() diff --git a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj index 09b595e08..e194182a0 100644 --- a/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj +++ b/tests/common/Smi.Common.Tests/Smi.Common.Tests.csproj @@ -32,9 +32,9 @@ - - - + + + diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs new file mode 100644 index 000000000..550f60559 --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnInfoValuesRejectorTests.cs @@ -0,0 +1,58 @@ +using FAnsi; +using FAnsi.Discovery; +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Moq; +using NUnit.Framework; +using Rdmp.Core.Curation; +using Rdmp.Core.Curation.Data; +using System.Collections.Generic; +using System.Data.Common; +using Tests.Common; + +namespace Microservices.CohortExtractor.Tests +{ + + public class ColumnInfoValuesRejectorTests : DatabaseTests + { + private const string PatColName = "PatientID"; + + [TestCase(DatabaseType.MicrosoftSQLServer)] + [TestCase(DatabaseType.MySql)] + public void Test_ColumnInfoValuesRejectorTests(DatabaseType type) + { + DiscoveredDatabase server = GetCleanedServer(type); + DiscoveredTable tbl = server.CreateTable("BadPatients", new[] { new DatabaseColumnRequest(PatColName, "varchar(100)") }); + + tbl.Insert(new Dictionary { { PatColName, "Frank" } }); + tbl.Insert(new Dictionary { { PatColName, "Peter" } }); + tbl.Insert(new Dictionary { { PatColName, "Frank" } }); //duplication for the lols + tbl.Insert(new Dictionary { { PatColName, "David" } }); + + new TableInfoImporter(CatalogueRepository, tbl).DoImport(out TableInfo _, out ColumnInfo[] cols); + + var rejector = new ColumnInfoValuesRejector(cols[0]); + + var moqDave = new Mock(); + moqDave.Setup(x => x[PatColName]) + .Returns("Dave"); + + Assert.IsFalse(rejector.Reject(moqDave.Object, out string reason)); + Assert.IsNull(reason); + + var moqFrank = new Mock(); + moqFrank.Setup(x => x[PatColName]) + .Returns("Frank"); + + Assert.IsTrue(rejector.Reject(moqFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + + var moqLowerCaseFrank = new Mock(); + moqLowerCaseFrank.Setup(x => x[PatColName]) + .Returns("frank"); + + Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + } + + } +} diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs new file mode 100644 index 000000000..2e317e716 --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/ColumnValuesRejectorTests.cs @@ -0,0 +1,55 @@ +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Moq; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Data.Common; + +namespace Microservices.CohortExtractor.Tests +{ + public class ColumnValuesRejectorTests + { + private const string PatColName = "PatientID"; + + [Test] + public void Test_ColumnValuesRejector_MissingColumn_Throws() + { + var rejector = new ColumnValuesRejector("fff",new HashSet{ "dave","frank"}); + + var moqDave = new Mock(); + moqDave + .Setup(x => x["fff"]) + .Throws(); + + var exc = Assert.Throws(() => rejector.Reject(moqDave.Object, out string _)); + Assert.True(exc.Message.Contains($"Expected a column called fff")); + } + + [Test] + public void Test_ColumnValuesRejectorTests() + { + var rejector = new ColumnValuesRejector(PatColName,new HashSet(new []{ "Frank","Peter","David"},StringComparer.CurrentCultureIgnoreCase)); + + var moqDave = new Mock(); + moqDave.Setup(x => x[PatColName]) + .Returns("Dave"); + + Assert.IsFalse(rejector.Reject(moqDave.Object, out string reason)); + Assert.IsNull(reason); + + var moqFrank = new Mock(); + moqFrank.Setup(x => x[PatColName]) + .Returns("Frank"); + + Assert.IsTrue(rejector.Reject(moqFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + + var moqLowerCaseFrank = new Mock(); + moqLowerCaseFrank.Setup(x => x[PatColName]) + .Returns("frank"); + + Assert.IsTrue(rejector.Reject(moqLowerCaseFrank.Object, out reason)); + Assert.AreEqual("Patient was in reject list", reason); + } + } +} diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs new file mode 100644 index 000000000..f5b38b7cf --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Execution/RequestFulfillers/QueryToExecuteResultTest.cs @@ -0,0 +1,54 @@ +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using NUnit.Framework; +using Smi.Common.Tests; +using System; + + +namespace Microservices.CohortExtractor.Tests.Execution.RequestFulfillers +{ + public class QueryToExecuteResultTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + /// + /// Asserts that we always have a rejection reason when rejection=true + /// + [Test] + public void Test_QueryToExecuteResult_RejectReasonNullOrEmpty_ThrowsException() + { + Assert.Throws(() => + { + var _ = new QueryToExecuteResult("foo", "bar", "baz", "whee", rejection: true, rejectionReason: null); + }); + Assert.Throws(() => + { + var _ = new QueryToExecuteResult("foo", "bar", "baz", "whee", rejection: true, rejectionReason: " "); + }); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs b/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs index 587a6987b..6e4815e62 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs +++ b/tests/microservices/Microservices.CohortExtractor.Tests/FromCataloguesExtractionRequestFulfillerTests.cs @@ -1,6 +1,9 @@  using FAnsi; +using FAnsi.Discovery; +using FAnsi.Extensions; using Microservices.CohortExtractor.Audit; +using Microservices.CohortExtractor.Execution; using Microservices.CohortExtractor.Execution.RequestFulfillers; using NUnit.Framework; using Rdmp.Core.Curation.Data; @@ -9,7 +12,6 @@ using System.Collections.Generic; using System.Data; using System.Linq; -using FAnsi.Extensions; using Tests.Common; namespace Microservices.CohortExtractor.Tests @@ -19,8 +21,13 @@ namespace Microservices.CohortExtractor.Tests /// (described in a ) and fetch matching image urls out of the database (creating /// ExtractImageCollection results). /// - class FromCataloguesExtractionRequestFulfillerTests : DatabaseTests + public class FromCataloguesExtractionRequestFulfillerTests : DatabaseTests { + [SetUp] + public void SetUp() + { + TestLogger.Setup(); + } [TestCase(DatabaseType.MicrosoftSQLServer), RequiresRelationalDb(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql), RequiresRelationalDb(DatabaseType.MySql)] @@ -30,7 +37,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching_SeriesInstan var dt = new DataTable(); dt.Columns.Add("SeriesInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); dt.Rows.Add("123", true, "/images/1.dcm"); @@ -56,7 +63,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching_SeriesInstan Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/2.dcm"))); } - + [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInstanceUIDOnly(DatabaseType databaseType) @@ -65,7 +72,7 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInsta var dt = new DataTable(); dt.Columns.Add("SeriesInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); dt.Rows.Add("123", true, "/images/1.dcm"); @@ -95,7 +102,7 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter_SeriesInsta Assert.AreEqual(1, matching[0].Accepted.Count); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); } - + [TestCase(DatabaseType.MicrosoftSQLServer), RequiresRelationalDb(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql), RequiresRelationalDb(DatabaseType.MySql)] @@ -110,10 +117,10 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching(DatabaseType dt.Columns.Add("Extractable"); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); - dt.Rows.Add("1.1","123.1","1.1", true, "/images/1.dcm"); - dt.Rows.Add("1.1","123.1","2.1", false, "/images/2.dcm"); - dt.Rows.Add("1.1","1234.1","3.1", false, "/images/3.dcm"); - dt.Rows.Add("1.1","1234.1","4.1", true, "/images/4.dcm"); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.Rows.Add("1.1", "123.1", "2.1", false, "/images/2.dcm"); + dt.Rows.Add("1.1", "1234.1", "3.1", false, "/images/3.dcm"); + dt.Rows.Add("1.1", "1234.1", "4.1", true, "/images/4.dcm"); dt.SetDoNotReType(true); @@ -133,6 +140,7 @@ public void FromCataloguesExtractionRequestFulfiller_NormalMatching(DatabaseType Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/2.dcm"))); } + [TestCase(DatabaseType.MicrosoftSQLServer)] [TestCase(DatabaseType.MySql)] public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseType databaseType) @@ -143,13 +151,13 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseTyp dt.Columns.Add("StudyInstanceUID"); dt.Columns.Add("SeriesInstanceUID"); dt.Columns.Add("SOPInstanceUID"); - dt.Columns.Add("Extractable",typeof(bool)); + dt.Columns.Add("Extractable", typeof(bool)); dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); - dt.Rows.Add("1.1","123.1","1.1", true, "/images/1.dcm"); - dt.Rows.Add("1.1","123.1","2.1", false, "/images/2.dcm"); - dt.Rows.Add("1.1","1234.1","3.1", false, "/images/3.dcm"); - dt.Rows.Add("1.1","1234.1","4.1", true, "/images/4.dcm"); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.Rows.Add("1.1", "123.1", "2.1", false, "/images/2.dcm"); + dt.Rows.Add("1.1", "1234.1", "3.1", false, "/images/3.dcm"); + dt.Rows.Add("1.1", "1234.1", "4.1", true, "/images/4.dcm"); dt.SetDoNotReType(true); @@ -173,5 +181,47 @@ public void FromCataloguesExtractionRequestFulfiller_MandatoryFilter(DatabaseTyp Assert.AreEqual(1, matching[0].Accepted.Count); Assert.AreEqual(1, matching[0].Accepted.Count(f => f.FilePathValue.Equals("/images/1.dcm"))); } + + [TestCase(DatabaseType.MicrosoftSQLServer, true)] + [TestCase(DatabaseType.MicrosoftSQLServer, false)] + public void Test_FromCataloguesExtractionRequestFulfiller_NoFilterExtraction(DatabaseType databaseType, bool isNoFiltersExtraction) + { + DiscoveredDatabase db = GetCleanedServer(databaseType); + + var dt = new DataTable(); + dt.Columns.Add("StudyInstanceUID"); + dt.Columns.Add("SeriesInstanceUID"); + dt.Columns.Add("SOPInstanceUID"); + dt.Columns.Add("Extractable", typeof(bool)); + dt.Columns.Add(QueryToExecuteColumnSet.DefaultImagePathColumnName); + dt.Rows.Add("1.1", "123.1", "1.1", true, "/images/1.dcm"); + dt.SetDoNotReType(true); + + DiscoveredTable tbl = db.CreateTable("FromCataloguesExtractionRequestFulfillerTests", dt); + Catalogue catalogue = Import(tbl); + + ExtractionInformation ei = catalogue.GetAllExtractionInformation(ExtractionCategory.Any).First(); + var filter = new ExtractionFilter(CatalogueRepository, "Extractable only", ei) + { + IsMandatory = true, + WhereSQL = "Extractable = 1" + }; + filter.SaveToDatabase(); + var fulfiller = new FromCataloguesExtractionRequestFulfiller(new[] { catalogue }); + fulfiller.Rejectors.Add(new RejectAll()); + + var message = new ExtractionRequestMessage + { + KeyTag = "SeriesInstanceUID", + ExtractionIdentifiers = new List(new [] { "123.1" }), + IsNoFilterExtraction = isNoFiltersExtraction, + }; + + ExtractImageCollection[] matching = fulfiller.GetAllMatchingFiles(message, new NullAuditExtractions()).ToArray(); + + int expected = isNoFiltersExtraction ? 1 : 0; + Assert.AreEqual(1, matching.Length); + Assert.AreEqual(expected, matching[0].Accepted.Count); + } } } diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs new file mode 100644 index 000000000..ebace6e1b --- /dev/null +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Messaging/ExtractionRequestQueueConsumerTest.cs @@ -0,0 +1,146 @@ +using Microservices.CohortExtractor.Audit; +using Microservices.CohortExtractor.Execution.ProjectPathResolvers; +using Microservices.CohortExtractor.Execution.RequestFulfillers; +using Microservices.CohortExtractor.Messaging; +using Moq; +using Newtonsoft.Json; +using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Framing; +using Smi.Common.Events; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using Smi.Common.Options; +using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; + + +namespace Microservices.CohortExtractor.Tests.Messaging +{ + public class ExtractionRequestQueueConsumerTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void Test_ExtractionRequestQueueConsumer_AnonExtraction_RoutingKey() + { + GlobalOptions globals = new GlobalOptionsFactory().Load(); + globals.CohortExtractorOptions.ExtractAnonRoutingKey = "anon"; + globals.CohortExtractorOptions.ExtractIdentRoutingKey = ""; + AssertMessagePublishedWithSpecifiedKey(globals, false, "anon"); + } + + [Test] + public void Test_ExtractionRequestQueueConsumer_IdentExtraction_RoutingKey() + { + GlobalOptions globals = new GlobalOptionsFactory().Load(); + globals.CohortExtractorOptions.ExtractAnonRoutingKey = ""; + globals.CohortExtractorOptions.ExtractIdentRoutingKey = "ident"; + AssertMessagePublishedWithSpecifiedKey(globals, true, "ident"); + } + + /// + /// Checks that ExtractionRequestQueueConsumer publishes messages correctly according to the input message isIdentifiableExtraction value + /// + /// + /// + /// + private static void AssertMessagePublishedWithSpecifiedKey(GlobalOptions globals, bool isIdentifiableExtraction, string expectedRoutingKey) + { + var fakeFulfiller = new FakeFulfiller(); + + var mockFileMessageProducerModel = new Mock(MockBehavior.Strict); + string fileMessageRoutingKey = null; + mockFileMessageProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsNotNull())) + .Callback((IMessage _, IMessageHeader __, string routingKey) => { fileMessageRoutingKey = routingKey; }) + .Returns(new MessageHeader()); + mockFileMessageProducerModel.Setup(x => x.WaitForConfirms()); + + var mockFileInfoMessageProducerModel = new Mock(MockBehavior.Strict); + mockFileInfoMessageProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsNotNull())) + .Returns(new MessageHeader()); + mockFileInfoMessageProducerModel.Setup(x => x.WaitForConfirms()); + + var msg = new ExtractionRequestMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "1234", + ExtractionDirectory = "1234/foo", + IsIdentifiableExtraction = isIdentifiableExtraction, + KeyTag = "foo", + ExtractionIdentifiers = new List { "foo" }, + Modality = null, + }; + var mockDeliverArgs = Mock.Of(MockBehavior.Strict); + mockDeliverArgs.DeliveryTag = 1; + mockDeliverArgs.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(msg)); + mockDeliverArgs.BasicProperties = new BasicProperties { Headers = new Dictionary() }; + var header = new MessageHeader(); + header.Populate(mockDeliverArgs.BasicProperties.Headers); + // Have to convert these to bytes since RabbitMQ normally does that when sending + mockDeliverArgs.BasicProperties.Headers["MessageGuid"] = Encoding.UTF8.GetBytes(header.MessageGuid.ToString()); + mockDeliverArgs.BasicProperties.Headers["ProducerExecutableName"] = Encoding.UTF8.GetBytes(header.ProducerExecutableName); + mockDeliverArgs.BasicProperties.Headers["Parents"] = Encoding.UTF8.GetBytes(string.Join("->", header.Parents)); + + var consumer = new ExtractionRequestQueueConsumer( + globals.CohortExtractorOptions, + fakeFulfiller, + new NullAuditExtractions(), new DefaultProjectPathResolver(), + mockFileMessageProducerModel.Object, + mockFileInfoMessageProducerModel.Object); + + var fatalCalled = false; + FatalErrorEventArgs fatalErrorEventArgs = null; + consumer.OnFatal += (sender, args) => + { + fatalCalled = true; + fatalErrorEventArgs = args; + }; + + var mockModel = new Mock(MockBehavior.Strict); + mockModel.Setup(x => x.IsClosed).Returns(false); + mockModel.Setup(x => x.BasicAck(It.IsAny(), It.IsAny())).Verifiable(); + + consumer.SetModel(mockModel.Object); + consumer.ProcessMessage(mockDeliverArgs); + + Thread.Sleep(500); // Fatal call is race-y + Assert.False(fatalCalled, $"Fatal was called with {fatalErrorEventArgs}"); + mockModel.Verify(x => x.BasicAck(It.IsAny(), It.IsAny()), Times.Once); + Assert.AreEqual(expectedRoutingKey, fileMessageRoutingKey); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj index cb8e2240c..e5407fb94 100644 --- a/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj +++ b/tests/microservices/Microservices.CohortExtractor.Tests/Microservices.CohortExtractor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs index dde9480b2..6a2ae7b0d 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/CohortPackagerHostTest.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Threading; using Microservices.CohortPackager.Execution; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.JobProcessing.Notifying; @@ -13,6 +10,9 @@ using Smi.Common.MongoDB; using Smi.Common.Options; using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Threading; namespace Microservices.CohortPackager.Tests.Execution { @@ -47,11 +47,8 @@ public void TearDown() { } private class TestReporter : IJobReporter { - public string Report { get; set; } - public void CreateReport(Guid jobId) - { - Report = $"Report for {jobId}"; - } + public bool reportCreated; + public void CreateReport(Guid jobId) => reportCreated = true; } private class TestLoggingNotifier : IJobCompleteNotifier @@ -65,7 +62,7 @@ public void NotifyJobCompleted(ExtractJobInfo jobInfo) } [Test] - public void TestCohortPackagerHost_HappyPath() + public void Test_CohortPackagerHost_HappyPath() { Guid jobId = Guid.NewGuid(); var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage @@ -95,21 +92,21 @@ public void TestCohortPackagerHost_HappyPath() }, KeyValue = "study-1", }; - var testExtractFileStatusMessage = new ExtractFileStatusMessage + var testExtractFileStatusMessage = new ExtractedFileStatusMessage { JobSubmittedAt = DateTime.UtcNow, - AnonymisedFileName = "study-1-anon-1.dcm", + OutputFilePath = "study-1-anon-1.dcm", ProjectNumber = "testProj1", ExtractionJobIdentifier = jobId, ExtractionDirectory = "test", - Status = ExtractFileStatus.ErrorWontRetry, + Status = ExtractedFileStatus.ErrorWontRetry, StatusMessage = "Couldn't anonymise", DicomFilePath = "study-1-orig-1.dcm", }; - var testIsIdentifiableMessage = new IsIdentifiableMessage + var testIsIdentifiableMessage = new ExtractedFileVerificationMessage { JobSubmittedAt = DateTime.UtcNow, - AnonymisedFileName = "study-1-anon-2.dcm", + OutputFilePath = "study-1-anon-2.dcm", ProjectNumber = "testProj1", ExtractionJobIdentifier = jobId, ExtractionDirectory = "test", @@ -119,7 +116,7 @@ public void TestCohortPackagerHost_HappyPath() }; - GlobalOptions globals = GlobalOptions.Load(); + GlobalOptions globals = new GlobalOptionsFactory().Load(); globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); @@ -129,12 +126,12 @@ public void TestCohortPackagerHost_HappyPath() globals.RabbitOptions, globals.CohortPackagerOptions.ExtractRequestInfoOptions, globals.CohortPackagerOptions.FileCollectionInfoOptions, - globals.CohortPackagerOptions.AnonFailedOptions, + globals.CohortPackagerOptions.NoVerifyStatusOptions, globals.CohortPackagerOptions.VerificationStatusOptions)) { tester.SendMessage(globals.CohortPackagerOptions.ExtractRequestInfoOptions, new MessageHeader(), testExtractionRequestInfoMessage); tester.SendMessage(globals.CohortPackagerOptions.FileCollectionInfoOptions, new MessageHeader(), testExtractFileCollectionInfoMessage); - tester.SendMessage(globals.CohortPackagerOptions.AnonFailedOptions, new MessageHeader(), testExtractFileStatusMessage); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage); tester.SendMessage(globals.CohortPackagerOptions.VerificationStatusOptions, new MessageHeader(), testIsIdentifiableMessage); var reporter = new TestReporter(); @@ -151,6 +148,101 @@ public void TestCohortPackagerHost_HappyPath() host.Stop("Test end"); Assert.True(notifier.JobCompleted && timeoutSecs >= 0); + Assert.True(reporter.reportCreated); + } + } + + [Test] + public void Test_CohortPackagerHost_IdentifiableExtraction() + { + Guid jobId = Guid.NewGuid(); + var testExtractionRequestInfoMessage = new ExtractionRequestInfoMessage + { + ExtractionModality = "MR", + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + KeyTag = "StudyInstanceUID", + KeyValueCount = 1, + IsIdentifiableExtraction = true, + }; + var testExtractFileCollectionInfoMessage = new ExtractFileCollectionInfoMessage + { + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + ExtractFileMessagesDispatched = new JsonCompatibleDictionary + { + { new MessageHeader(), "out1.dcm" }, + { new MessageHeader(), "out2.dcm" }, + }, + RejectionReasons = new Dictionary + { + {"rejected - blah", 1 }, + }, + KeyValue = "study-1", + IsIdentifiableExtraction = true, + }; + var testExtractFileStatusMessage1 = new ExtractedFileStatusMessage + { + JobSubmittedAt = DateTime.UtcNow, + OutputFilePath = "src.dcm", + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + Status = ExtractedFileStatus.Copied, + StatusMessage = null, + DicomFilePath = "study-1-orig-1.dcm", + IsIdentifiableExtraction = true, + }; + var testExtractFileStatusMessage2 = new ExtractedFileStatusMessage + { + JobSubmittedAt = DateTime.UtcNow, + OutputFilePath = "src_missing.dcm", + ProjectNumber = "testProj1", + ExtractionJobIdentifier = jobId, + ExtractionDirectory = "test", + Status = ExtractedFileStatus.FileMissing, + StatusMessage = null, + DicomFilePath = "study-1-orig-2.dcm", + IsIdentifiableExtraction = true, + }; + + GlobalOptions globals = new GlobalOptionsFactory().Load(); + globals.CohortPackagerOptions.JobWatcherTimeoutInSeconds = 5; + + MongoClient client = MongoClientHelpers.GetMongoClient(globals.MongoDatabases.ExtractionStoreOptions, "test", true); + client.DropDatabase(globals.MongoDatabases.ExtractionStoreOptions.DatabaseName); + + using (var tester = new MicroserviceTester( + globals.RabbitOptions, + globals.CohortPackagerOptions.ExtractRequestInfoOptions, + globals.CohortPackagerOptions.FileCollectionInfoOptions, + globals.CohortPackagerOptions.NoVerifyStatusOptions, + globals.CohortPackagerOptions.VerificationStatusOptions)) + { + tester.SendMessage(globals.CohortPackagerOptions.ExtractRequestInfoOptions, new MessageHeader(), testExtractionRequestInfoMessage); + tester.SendMessage(globals.CohortPackagerOptions.FileCollectionInfoOptions, new MessageHeader(), testExtractFileCollectionInfoMessage); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage1); + tester.SendMessage(globals.CohortPackagerOptions.NoVerifyStatusOptions, new MessageHeader(), testExtractFileStatusMessage2); + + var reporter = new TestReporter(); + var notifier = new TestLoggingNotifier(); + var host = new CohortPackagerHost(globals, reporter, notifier, null, false); + host.Start(); + + var timeoutSecs = 30; + while (!notifier.JobCompleted && timeoutSecs > 0) + { + --timeoutSecs; + Thread.Sleep(TimeSpan.FromSeconds(1)); + } + + host.Stop("Test end"); + Assert.True(notifier.JobCompleted && timeoutSecs >= 0); + Assert.True(reporter.reportCreated); } } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs index 16a6bd0b0..730167c8f 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobInfoTest.cs @@ -1,7 +1,7 @@ -using System; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using NUnit.Framework; using Smi.Common.Tests; +using System; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage { @@ -48,7 +48,10 @@ public void TestExtractJobInfo_Equality() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -57,7 +60,10 @@ public void TestExtractJobInfo_Equality() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(info1, info2); } @@ -74,7 +80,10 @@ public void TestExtractJobInfo_GetHashCode() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); var info2 = new ExtractJobInfo( guid, _dateTimeProvider.UtcNow(), @@ -83,7 +92,10 @@ public void TestExtractJobInfo_GetHashCode() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(info1.GetHashCode(), info2.GetHashCode()); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs index b99d67df8..5b7a5e78e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/ExtractJobStoreTest.cs @@ -1,10 +1,10 @@ -using System; -using System.Collections.Generic; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using NUnit.Framework; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Collections.Generic; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage @@ -34,8 +34,8 @@ private class TestExtractJobStore : ExtractJobStore { protected override void PersistMessageToStoreImpl(ExtractionRequestInfoMessage message, IMessageHeader header) { } protected override void PersistMessageToStoreImpl(ExtractFileCollectionInfoMessage collectionInfoMessage, IMessageHeader header) => throw new NotImplementedException(); - protected override void PersistMessageToStoreImpl(ExtractFileStatusMessage message, IMessageHeader header) { } - protected override void PersistMessageToStoreImpl(IsIdentifiableMessage message, IMessageHeader header) { } + protected override void PersistMessageToStoreImpl(ExtractedFileStatusMessage message, IMessageHeader header) { } + protected override void PersistMessageToStoreImpl(ExtractedFileVerificationMessage message, IMessageHeader header) { } protected override List GetReadyJobsImpl(Guid specificJobId = new Guid()) => throw new NotImplementedException(); protected override void CompleteJobImpl(Guid jobId) { } protected override void MarkJobFailedImpl(Guid jobId, Exception e) { } @@ -43,6 +43,7 @@ protected override void MarkJobFailedImpl(Guid jobId, Exception e) { } protected override IEnumerable>> GetCompletedJobRejectionsImpl(Guid jobId) => throw new NotImplementedException(); protected override IEnumerable> GetCompletedJobAnonymisationFailuresImpl(Guid jobId) => throw new NotImplementedException(); protected override IEnumerable> GetCompletedJobVerificationFailuresImpl(Guid jobId) => throw new NotImplementedException(); + protected override IEnumerable GetCompletedJobMissingFileListImpl(Guid jobId) => new[] { "missing" }; } #endregion @@ -77,16 +78,16 @@ public void TestPersistMessageToStore_ExtractionRequestInfoMessage() public void TestPersistMessageToStore_ExtractFileStatusMessage() { var testExtractJobStore = new TestExtractJobStore(); - var message = new ExtractFileStatusMessage(); + var message = new ExtractedFileStatusMessage(); var header = new MessageHeader(); - message.Status = ExtractFileStatus.Unknown; + message.Status = ExtractedFileStatus.None; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); - message.Status = ExtractFileStatus.Anonymised; + message.Status = ExtractedFileStatus.Anonymised; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); - message.Status = ExtractFileStatus.ErrorWontRetry; + message.Status = ExtractedFileStatus.ErrorWontRetry; testExtractJobStore.PersistMessageToStore(message, header); } @@ -98,19 +99,19 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() var header = new MessageHeader(); // Must have AnonymisedFileName - var message = new IsIdentifiableMessage(); - message.AnonymisedFileName = ""; + var message = new ExtractedFileVerificationMessage(); + message.OutputFilePath = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report shouldn't be an empty string or null - message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message = new ExtractedFileVerificationMessage(); + message.OutputFilePath = "anon.dcm"; message.Report = ""; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); // Report needs to contain content if marked as IsIdentifiable - message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message = new ExtractedFileVerificationMessage(); + message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = true; message.Report = "[]"; Assert.Throws(() => testExtractJobStore.PersistMessageToStore(message, header)); @@ -119,8 +120,8 @@ public void TestPersistMessageToStore_IsIdentifiableMessage() testExtractJobStore.PersistMessageToStore(message, header); // Report can be empty if not marked as IsIdentifiable - message = new IsIdentifiableMessage(); - message.AnonymisedFileName = "anon.dcm"; + message = new ExtractedFileVerificationMessage(); + message.OutputFilePath = "anon.dcm"; message.IsIdentifiable = false; message.Report = "[]"; testExtractJobStore.PersistMessageToStore(message, header); @@ -147,6 +148,14 @@ public void TestMarkJobFailed() store.MarkJobFailed(Guid.NewGuid(), new Exception()); } + [Test] + public void Test_GetCompletedJobMissingFileList() + { + var store = new TestExtractJobStore(); + Assert.Throws(() => store.GetCompletedJobMissingFileList(default)); + Assert.AreEqual(new[] { "missing" }, store.GetCompletedJobMissingFileList(Guid.NewGuid())); + } + #endregion } } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs index 25764fed7..7c56cdadb 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobInfoExtensionsTest.cs @@ -1,5 +1,4 @@ -using System; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; using NUnit.Framework; @@ -7,6 +6,7 @@ using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB @@ -59,6 +59,8 @@ public void TestToExtractJobInfo() ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, + IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); @@ -72,7 +74,10 @@ public void TestToExtractJobInfo() "KeyTag", 123, "MR", - ExtractJobStatus.WaitingForCollectionInfo); + ExtractJobStatus.WaitingForCollectionInfo, + isIdentifiableExtraction: true, + isNoFilterExtraction: true + ); Assert.AreEqual(expected, extractJobInfo); } diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs index 76de61864..d7e89e68e 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/MongoExtractJobStoreTest.cs @@ -1,7 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; @@ -16,6 +12,11 @@ using Smi.Common.MongoDb.Tests; using Smi.Common.MongoDB.Tests; using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB { @@ -224,6 +225,8 @@ public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() KeyTag = "StudyInstanceUID", KeyValueCount = 1, ExtractionModality = "CT", + IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; var testHeader = new MessageHeader { @@ -253,6 +256,8 @@ public void TestPersistMessageToStoreImpl_ExtractionRequestInfoMessage() "StudyInstanceUID", 1, "CT", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(expected, extractJob); @@ -359,11 +364,11 @@ public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); - var testExtractFileStatusMessage = new ExtractFileStatusMessage + var testExtractFileStatusMessage = new ExtractedFileStatusMessage { - AnonymisedFileName = "anon.dcm", + OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), - Status = ExtractFileStatus.ErrorWontRetry, + Status = ExtractedFileStatus.ErrorWontRetry, ProjectNumber = "1234", ExtractionJobIdentifier = jobId, ExtractionDirectory = "1234/test", @@ -374,16 +379,17 @@ public void TestPersistMessageToStoreImpl_ExtractFileStatusMessage() store.PersistMessageToStore(testExtractFileStatusMessage, header); - Dictionary docs = client.ExtractionDatabase.StatusCollections[$"statuses_{jobId}"].Documents; Assert.AreEqual(docs.Count, 1); MongoFileStatusDoc statusDoc = docs.Values.ToList()[0]; var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), + "original.dcm", "anon.dcm", false, true, + ExtractedFileStatus.ErrorWontRetry, "Could not anonymise"); Assert.True(statusDoc.Equals(expected)); @@ -396,9 +402,9 @@ public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() var store = new MongoExtractJobStore(client, ExtractionDatabaseName, _dateTimeProvider); Guid jobId = Guid.NewGuid(); - var testIsIdentifiableMessage = new IsIdentifiableMessage + var testIsIdentifiableMessage = new ExtractedFileVerificationMessage { - AnonymisedFileName = "anon.dcm", + OutputFilePath = "anon.dcm", JobSubmittedAt = _dateTimeProvider.UtcNow(), ProjectNumber = "1234", ExtractionJobIdentifier = jobId, @@ -417,9 +423,11 @@ public void TestPersistMessageToStoreImpl_IsIdentifiableMessage() var expected = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, header, _dateTimeProvider), + "original.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "[]"); Assert.True(statusDoc.Equals(expected)); @@ -439,6 +447,8 @@ public void TestGetReadJobsImpl() "SeriesInstanceUID", 1, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -453,9 +463,11 @@ public void TestGetReadJobsImpl() ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), + "input.dcm", "anon1.dcm", true, false, + ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); @@ -514,6 +526,8 @@ public void TestCompleteJobImpl() "SeriesInstanceUID", 1, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var testMongoExpectedFilesDoc = new MongoExpectedFilesDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), @@ -528,9 +542,11 @@ public void TestCompleteJobImpl() ); var testMongoFileStatusDoc = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(jobId, new MessageHeader(), _dateTimeProvider), + "input.dcm", "anon1.dcm", true, false, + ExtractedFileStatus.Anonymised, "Verified"); var client = new TestMongoClient(); @@ -620,6 +636,8 @@ public void TestMarkJobFailedImpl() "1.2.3.4", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var client = new TestMongoClient(); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs index 27cbf6f42..996f7d3c5 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoCompletedExtractJobDocTest.cs @@ -1,11 +1,13 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel { @@ -24,6 +26,8 @@ public class MongoCompletedExtractJobDocTest "test", 1, null, + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); #region Fixture Methods @@ -47,13 +51,47 @@ public void TearDown() { } #region Tests + [Test] + public void Test_MongoCompletedExtractJobDoc_ParseOldFormat() + { + Console.WriteLine(Guid.NewGuid()); + const string jsonDoc = @" +{ + '_id' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'header' : { + 'extractionJobIdentifier' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'messageGuid' : 'bfead735-d5c0-4f7c-b0a7-88d873704dab', + 'producerExecutableName' : 'ExtractorCL', + 'producerProcessID' : 1234, + 'originalPublishTimestamp' : ISODate('2020-08-28T12:00:00Z'), + 'parents' : '', + 'receivedAt' : ISODate('2020-08-28T12:00:00Z') + }, + 'projectNumber' : '1234s', + 'jobStatus' : 'Completed', + 'extractionDirectory' : 'foo/bar', + 'jobSubmittedAt' : ISODate('2020-08-28T12:00:00Z'), + 'keyTag' : 'SeriesInstanceUID', + 'keyCount' : 123, + 'extractionModality' : null, + 'failedJobInfo' : null, + 'completedAt' : ISODate('2020-08-28T12:00:00Z'), +}"; + + var mongoExtractJobDoc = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + // NOTE(rkm 2020-08-28) This works by chance since the missing bool will default to false, so we don't require MongoCompletedExtractJobDoc to implement ISupportInitialize + Assert.False(mongoExtractJobDoc.IsIdentifiableExtraction); + Assert.False(mongoExtractJobDoc.IsNoFilterExtraction); + } + [Test] public void TestMongoCompletedExtractJobDoc_SettersAvailable() { foreach (PropertyInfo p in typeof(MongoCompletedExtractJobDoc).GetProperties()) Assert.True(p.CanWrite, $"Property '{p.Name}' is not writeable"); } - + [Test] public void TestMongoCompletedExtractJobDoc_Constructor_ExtractJobStatus() { diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs index 98ba37c8d..61a46d786 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoExtractJobDocTest.cs @@ -1,12 +1,12 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage; +using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; using NUnit.Framework; using Smi.Common.Helpers; using Smi.Common.Messages; using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -62,6 +62,8 @@ public void TestMongoExtractJobDoc_FromMessage() ExtractionDirectory = "test/directory", KeyTag = "KeyTag", KeyValueCount = 123, + IsIdentifiableExtraction = true, + IsNoFilterExtraction = true, }; MongoExtractJobDoc doc = MongoExtractJobDoc.FromMessage(message, _messageHeader, _dateTimeProvider); @@ -76,6 +78,8 @@ public void TestMongoExtractJobDoc_FromMessage() "KeyTag", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(expected, doc); @@ -97,6 +101,8 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, failedInfoDoc); var doc2 = new MongoExtractJobDoc( guid, @@ -108,6 +114,8 @@ public void TestMongoExtractJobDoc_Equality() "KeyTag", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, failedInfoDoc); Assert.AreEqual(doc1, doc2); @@ -128,6 +136,8 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); var doc2 = new MongoExtractJobDoc( guid, @@ -139,6 +149,8 @@ public void TestMongoExtractJobDoc_GetHashCode() "KeyTag", 123, "MR", + isIdentifiableExtraction: true, + isNoFilterExtraction: true, null); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs index 862a03a46..6b3d53337 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/ExtractJobStorage/MongoDB/ObjectModel/MongoFileStatusDocTest.cs @@ -1,9 +1,13 @@ -using System; -using System.Reflection; -using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using Microservices.CohortPackager.Execution.ExtractJobStorage.MongoDB.ObjectModel; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; using NUnit.Framework; +using Smi.Common.Helpers; using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; using Smi.Common.Tests; +using System; +using System.Reflection; namespace Microservices.CohortPackager.Tests.Execution.ExtractJobStorage.MongoDB.ObjectModel @@ -39,6 +43,85 @@ public void TearDown() { } #region Tests + [Test] + public void Test_MongoFileStatusDoc_IsIdentifiable_StatusMessage() + { + Assert.Throws(() => + new MongoFileStatusDoc( + MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()), + "input.dcm", + "anon.dcm", + true, + false, + ExtractedFileStatus.Anonymised, + null)); + Assert.DoesNotThrow(() => + new MongoFileStatusDoc( + MongoExtractionMessageHeaderDoc.FromMessageHeader(Guid.NewGuid(), new MessageHeader(), new DateTimeProvider()), + "input.dcm", + "anon.dcm", + true, + true, + ExtractedFileStatus.Anonymised, + null)); + } + + [Test] + public void Test_MongoFileStatusDoc_ParseOldFormat_VerificationMessage() + { + // NOTE(rkm 2020-08-28) Format as of release v1.11.1 + const string jsonDoc = @" +{ + '_id' : ObjectId('5f490ef8473b9739448cbe4c'), + 'header': { + 'extractionJobIdentifier':'f9586843-8dbb-46a6-b36d-4646fdfddede', + 'messageGuid': '21a63ac3-c6f0-4fb9-973c-c97490920246', + 'producerExecutableName':'IsIdentifiable', + 'producerProcessID': 1234, + 'originalPublishTimestamp': ISODate('2020-08-28T12:00:00.000Z'), + 'parents': 'cd6430dc-952e-420e-808c-7910e61e9278->a9e16701-ef8b-482c-8b1b-023f6f40fdde->cc84ebbc-ebd0-40d0-a7da-8a2c5004b8bc', + 'receivedAt': ISODate('2020-08-28T12:00:00.000Z') + }, + 'anonymisedFileName' : 'anon.dcm', + 'wasAnonymised' : true, + 'isIdentifiable' : false, + 'statusMessage' : '[]' +}"; + var parsed = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + Assert.AreEqual("anon.dcm", parsed.OutputFileName); + Assert.AreEqual("", parsed.DicomFilePath); + Assert.AreEqual(ExtractedFileStatus.Anonymised, parsed.ExtractedFileStatus); + } + + [Test] + public void Test_MongoFileStatusDoc_ParseOldFormat_AnonFailedMessage() + { + // NOTE(rkm 2020-08-28) Format as of release v1.11.1 + const string jsonDoc = @" +{ + '_id' : ObjectId('5f490ef8473b9739448cbe4c'), + 'header': { + 'extractionJobIdentifier':'f9586843-8dbb-46a6-b36d-4646fdfddede', + 'messageGuid': '21a63ac3-c6f0-4fb9-973c-c97490920246', + 'producerExecutableName':'CTPAnonymiser', + 'producerProcessID': 1234, + 'originalPublishTimestamp': ISODate('2020-08-28T12:00:00.000Z'), + 'parents': 'cd6430dc-952e-420e-808c-7910e61e9278->a9e16701-ef8b-482c-8b1b-023f6f40fdde', + 'receivedAt': ISODate('2020-08-28T12:00:00.000Z') + }, + 'anonymisedFileName' : null, + 'wasAnonymised' : false, + 'isIdentifiable' : false, + 'statusMessage' : 'failed to anonymise' +}"; + var parsed = BsonSerializer.Deserialize(BsonDocument.Parse(jsonDoc)); + + Assert.AreEqual(null, parsed.OutputFileName); + Assert.AreEqual("", parsed.DicomFilePath); + Assert.AreEqual(ExtractedFileStatus.ErrorWontRetry, parsed.ExtractedFileStatus); + } + [Test] public void TestMongoFileStatusDoc_SettersAvailable() { @@ -52,16 +135,20 @@ public void TestMongoFileStatusDoc_Equality() Guid guid = Guid.NewGuid(); var doc1 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); var doc2 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); Assert.AreEqual(doc1, doc2); @@ -73,16 +160,20 @@ public void TestMongoFileStatusDoc_GetHashCode() Guid guid = Guid.NewGuid(); var doc1 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); var doc2 = new MongoFileStatusDoc( MongoExtractionMessageHeaderDoc.FromMessageHeader(guid, _messageHeader, _dateTimeProvider), + "input.dcm", "anon.dcm", true, false, + ExtractedFileStatus.Anonymised, "anonymised"); Assert.AreEqual(doc1.GetHashCode(), doc2.GetHashCode()); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs index f81d19686..3e22c8924 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/ExtractJobWatcherTest.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using Microservices.CohortPackager.Execution.ExtractJobStorage; using Microservices.CohortPackager.Execution.JobProcessing; using Microservices.CohortPackager.Execution.JobProcessing.Notifying; @@ -8,6 +6,8 @@ using NUnit.Framework; using Smi.Common.Options; using Smi.Common.Tests; +using System; +using System.Collections.Generic; namespace Microservices.CohortPackager.Tests.Execution.JobProcessing { @@ -70,7 +70,9 @@ public void TestProcessJobs() "KeyTag", 123, null, - ExtractJobStatus.ReadyForChecks + ExtractJobStatus.ReadyForChecks, + isIdentifiableExtraction: true, + isNoFilterExtraction: true ); var opts = new CohortPackagerOptions { JobWatcherTimeoutInSeconds = 123 }; diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs similarity index 57% rename from tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs rename to tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs index 7871d4d78..b5f562564 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterTest.cs +++ b/tests/microservices/Microservices.CohortPackager.Tests/Execution/JobProcessing/Reporting/JobReporterBaseTest.cs @@ -12,7 +12,7 @@ namespace Microservices.CohortPackager.Tests.Execution.JobProcessing.Reporting { [TestFixture] - public class JobReporterTest + public class JobReporterBaseTest { #region Fixture Methods @@ -42,13 +42,9 @@ private class TestJobReporter : JobReporterBase public bool Disposed { get; set; } - public TestJobReporter(IExtractJobStore jobStore) - : base(jobStore, null) { } + public TestJobReporter(IExtractJobStore jobStore) : base(jobStore, null) { } - protected override Stream GetStream(Guid jobId) - { - return new MemoryStream(); - } + protected override Stream GetStream(Guid jobId) => new MemoryStream(); protected override void FinishReport(Stream stream) { @@ -58,17 +54,8 @@ protected override void FinishReport(Stream stream) } } - - protected override void ReleaseUnmanagedResources() - { - Disposed = true; - } - - - public override void Dispose() - { - ReleaseUnmanagedResources(); - } + protected override void ReleaseUnmanagedResources() => Disposed = true; + public override void Dispose() => ReleaseUnmanagedResources(); } [Test] @@ -84,7 +71,10 @@ public void Test_JobReporterBase_CreateReport_Empty() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var mockJobStore = new Mock(MockBehavior.Strict); mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); @@ -107,18 +97,22 @@ public void Test_JobReporterBase_CreateReport_Empty() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary -Full details: +### Full details ## Rejected files @@ -146,7 +140,10 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var rejections = new List>> { @@ -201,21 +198,25 @@ public void Test_JobReporterBase_CreateReport_WithBasicData() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary - Tag: ScanOptions (1 total occurrence(s)) - Value: 'FOO' (1 occurrence(s)) -Full details: +### Full details - Tag: ScanOptions (1 total occurrence(s)) - Value: 'FOO' (1 occurrence(s)) @@ -252,7 +253,10 @@ public void Test_JobReporterBase_WriteJobVerificationFailures_JsonException() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var verificationFailures = new List> { @@ -287,7 +291,10 @@ public void Test_JobReporterBase_CreateReport_AggregateData() "keyTag", 123, "ZZ", - ExtractJobStatus.Completed); + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); var verificationFailures = new List> { @@ -370,15 +377,19 @@ public void Test_JobReporterBase_CreateReport_AggregateData() - Extraction tag: keyTag - Extraction modality: ZZ - Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: Yes Report contents: - Verification failures + - Summary + - Full Details - Rejected failures - Anonymisation failures ## Verification failures -Summary: +### Summary - Tag: ScanOptions (3 total occurrence(s)) - Value: 'FOO' (2 occurrence(s)) @@ -388,7 +399,7 @@ public void Test_JobReporterBase_CreateReport_AggregateData() - Value: 'BAZ' (2 occurrence(s)) -Full details: +### Full details - Tag: ScanOptions (3 total occurrence(s)) - Value: 'FOO' (2 occurrence(s)) @@ -409,6 +420,260 @@ public void Test_JobReporterBase_CreateReport_AggregateData() ## Anonymisation failures +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } + + [Test] + public void Test_JobReporterBase_CreateReport_WithPixelData() + { + // NOTE(rkm 2020-08-25) Tests that the "Z" tag is ordered before PixelData, and that PixelData items are ordered by decreasing length not by occurrence + + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: false + ); + + const string report = @" +[ + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'aaaaaaaaaaa' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'a' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'PixelData', + 'ProblemValue': 'a' + }, + { + 'Parts': [], + 'Resource': 'unused', + 'ResourcePrimaryKey': 'unused', + 'ProblemField': 'Z', + 'ProblemValue': 'bar' + }, +]"; + + var verificationFailures = new List> + { + new Tuple("foo1.dcm", report), + }; + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobRejections(It.IsAny())).Returns(new List>>()); + mockJobStore.Setup(x => x.GetCompletedJobAnonymisationFailures(It.IsAny())).Returns(new List>()); + mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())).Returns(verificationFailures); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: Yes + +Report contents: +- Verification failures + - Summary + - Full Details +- Rejected failures +- Anonymisation failures + +## Verification failures + +### Summary + +- Tag: Z (1 total occurrence(s)) + - Value: 'bar' (1 occurrence(s)) + +- Tag: PixelData (3 total occurrence(s)) + - Value: 'aaaaaaaaaaa' (1 occurrence(s)) + - Value: 'a' (2 occurrence(s)) + + +### Full details + +- Tag: Z (1 total occurrence(s)) + - Value: 'bar' (1 occurrence(s)) + - foo1.dcm + +- Tag: PixelData (3 total occurrence(s)) + - Value: 'aaaaaaaaaaa' (1 occurrence(s)) + - foo1.dcm + - Value: 'a' (2 occurrence(s)) + - foo1.dcm + - foo1.dcm + + +## Rejected files + + +## Anonymisation failures + + +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } + + [Test] + public void Test_JobReporterBase_CreateReport_IdentifiableExtraction() + { + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed, + isIdentifiableExtraction: true, + isNoFilterExtraction: false + ); + + var missingFiles = new List + { + "missing.dcm", + }; + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobMissingFileList(It.IsAny())).Returns(missingFiles); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 +- Identifiable extraction: Yes +- Filtered extraction: Yes + +Report contents: +- Missing file list (files which were selected from an input ID but could not be found) + +## Missing file list + +- missing.dcm + +--- end of report --- +"; + TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); + Assert.True(reporter.Disposed); + } + + + [Test] + public void Test_JobReporterBase_CreateReport_FilteredExtraction() + { + Guid jobId = Guid.NewGuid(); + var provider = new TestDateTimeProvider(); + var testJobInfo = new ExtractJobInfo( + jobId, + provider.UtcNow(), + "1234", + "test/dir", + "keyTag", + 123, + "ZZ", + ExtractJobStatus.Completed, + isIdentifiableExtraction: false, + isNoFilterExtraction: true + ); + + var mockJobStore = new Mock(MockBehavior.Strict); + mockJobStore.Setup(x => x.GetCompletedJobInfo(It.IsAny())).Returns(testJobInfo); + mockJobStore.Setup(x => x.GetCompletedJobRejections(It.IsAny())).Returns(new List>>()); + mockJobStore.Setup(x => x.GetCompletedJobAnonymisationFailures(It.IsAny())).Returns(new List>()); + mockJobStore.Setup(x => x.GetCompletedJobVerificationFailures(It.IsAny())).Returns(new List>()); + + TestJobReporter reporter; + using (reporter = new TestJobReporter(mockJobStore.Object)) + { + reporter.CreateReport(Guid.Empty); + } + + string expected = $@" +# SMI file extraction report for 1234 + +Job info: +- Job submitted at: {provider.UtcNow().ToString("s", CultureInfo.InvariantCulture)} +- Job extraction id: {jobId} +- Extraction tag: keyTag +- Extraction modality: ZZ +- Requested identifier count: 123 +- Identifiable extraction: No +- Filtered extraction: No + +Report contents: +- Verification failures + - Summary + - Full Details +- Rejected failures +- Anonymisation failures + +## Verification failures + +### Summary + + +### Full details + + +## Rejected files + + +## Anonymisation failures + + --- end of report --- "; TestHelpers.AreEqualIgnoringCaseAndLineEndings(expected, reporter.Report); diff --git a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj index cfefe690f..fb59450aa 100644 --- a/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj +++ b/tests/microservices/Microservices.CohortPackager.Tests/Microservices.CohortPackager.Tests.csproj @@ -13,13 +13,13 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj index 1460802e2..86290491d 100644 --- a/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj +++ b/tests/microservices/Microservices.DeadLetterReprocessor.Tests/Microservices.DeadLetterReprocessor.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd b/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd deleted file mode 100644 index f4e0ad88b..000000000 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/AllServices.cd +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs index c1e7e7398..0ca26a4e5 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/HowFastIsDLETest.cs @@ -76,7 +76,7 @@ public void TestLargeImageDatasets(DatabaseType databaseType, int numberOfImages var template = ImageTableTemplateCollection.LoadFrom(_templateXml); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; @@ -158,7 +158,7 @@ public void TestBulkInsertOnly(DatabaseType databaseType, int numberOfImages) var template = ImageTableTemplateCollection.LoadFrom(_templateXml); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.DicomRelationalMapperOptions.DatabaseNamerType = typeof(MyFixedStagingDatabaseNamer).FullName; _globals.DicomRelationalMapperOptions.QoSPrefetchCount = ushort.MaxValue; diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs index 1f1bedbd4..92d3f629f 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperHostTests.cs @@ -31,7 +31,7 @@ public void TestCreatingNamer_CorrectType(DatabaseType dbType, string typeName, var cata = Import(tbl); - var globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var consumerOptions = globals.DicomRelationalMapperOptions; var lmd = new LoadMetadata(CatalogueRepository, "MyLoad"); diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs index a74781a2d..eb566f900 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTests.cs @@ -34,7 +34,7 @@ public void Setup() { BlitzMainDataTables(); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); _helper = new DicomRelationalMapperTestHelper(); _helper.SetupSuite(db, RepositoryLocator, _globals, typeof(DicomDatasetCollectionSource)); diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj index d50305355..b5400cc61 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/Microservices.DicomRelationalMapper.Tests.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs index 2bb461f77..2cb9c258f 100644 --- a/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs +++ b/tests/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs @@ -57,7 +57,7 @@ public void SetupSuite(DiscoveredDatabase server, bool persistentRaw = false, st { TestLogger.Setup(); - _globals = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _globals = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); _globals.UseTestValues( RequiresRabbit.GetConnectionFactory(), @@ -461,14 +461,15 @@ private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Actiontrue - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs b/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs index c43358e85..fd2808608 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs +++ b/tests/microservices/Microservices.DicomTagReader.Tests/DicomTagReaderTestHelper.cs @@ -44,8 +44,8 @@ public void SetUpSuite() // Create the test Series/Image exchanges Options.RabbitOptions.RabbitMqControlExchangeName = "TEST.ControlExchange"; var tester = new MicroserviceTester(Options.RabbitOptions); - tester.CreateExchange(Options.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, new ConsumerOptions { QueueName = TestSeriesQueueName }); - tester.CreateExchange(Options.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, new ConsumerOptions { QueueName = TestImageQueueName }); + tester.CreateExchange(Options.DicomTagReaderOptions.ImageProducerOptions.ExchangeName, TestSeriesQueueName ); + tester.CreateExchange(Options.DicomTagReaderOptions.SeriesProducerOptions.ExchangeName, TestImageQueueName ); tester.CreateExchange(Options.RabbitOptions.FatalLoggingExchange, null); tester.CreateExchange(Options.RabbitOptions.RabbitMqControlExchangeName, null); tester.Shutdown(); @@ -71,7 +71,7 @@ public void ResetSuite() private void SetUpDefaults() { - Options = GlobalOptions.Load("default", TestContext.CurrentContext.TestDirectory); + Options = new GlobalOptionsFactory().Load("default", TestContext.CurrentContext.TestDirectory); AccessionConsumerOptions = Options.DicomTagReaderOptions; diff --git a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj index 5fc19dfa1..2dc6c0e52 100644 --- a/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj +++ b/tests/microservices/Microservices.DicomTagReader.Tests/Microservices.DicomTagReader.Tests.csproj @@ -13,12 +13,12 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs new file mode 100644 index 000000000..0fe1bf683 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierHostTest.cs @@ -0,0 +1,94 @@ +using Microservices.FileCopier.Execution; +using Newtonsoft.Json; +using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using Smi.Common.Messages.Extraction; +using Smi.Common.Options; +using Smi.Common.Tests; +using System; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Text; + + +namespace Microservices.FileCopier.Tests.Execution +{ + public class FileCopierHostTest + { + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() { } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void Test_FileCopierHost_HappyPath() + { + GlobalOptions globals = new GlobalOptionsFactory().Load(); + globals.FileSystemOptions.FileSystemRoot = "root"; + globals.FileSystemOptions.ExtractRoot = "exroot"; + + using var tester = new MicroserviceTester(globals.RabbitOptions, globals.FileCopierOptions); + + string outputQueueName = globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName.Replace("Exchange", "Queue"); + tester.CreateExchange( + globals.FileCopierOptions.CopyStatusProducerOptions.ExchangeName, + outputQueueName, + false, + globals.FileCopierOptions.NoVerifyRoutingKey); + + var mockFileSystem = new MockFileSystem(); + mockFileSystem.AddDirectory(globals.FileSystemOptions.FileSystemRoot); + mockFileSystem.AddDirectory(globals.FileSystemOptions.ExtractRoot); + mockFileSystem.AddFile(mockFileSystem.Path.Combine(globals.FileSystemOptions.FileSystemRoot, "file.dcm"), MockFileData.NullObject); + + var host = new FileCopierHost(globals, mockFileSystem, false); + tester.StopOnDispose.Add(host); + host.Start(); + + var message = new ExtractFileMessage + { + ExtractionJobIdentifier = Guid.NewGuid(), + JobSubmittedAt = DateTime.UtcNow, + ProjectNumber = "1234", + ExtractionDirectory = "1234/foo", + DicomFilePath = "file.dcm", + IsIdentifiableExtraction = true, + OutputPath = "output.dcm", + }; + tester.SendMessage(globals.FileCopierOptions, message); + + using IConnection conn = tester.Factory.CreateConnection(); + using IModel model = conn.CreateModel(); + var consumer = new EventingBasicConsumer(model); + ExtractedFileStatusMessage statusMessage = null; + consumer.Received += (_, ea) => statusMessage = JsonConvert.DeserializeObject(Encoding.UTF8.GetString(ea.Body.ToArray())); + model.BasicConsume(outputQueueName, true, "", consumer); + + new TestTimelineAwaiter().Await(() => statusMessage != null); + Assert.AreEqual(ExtractedFileStatus.Copied, statusMessage.Status); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs new file mode 100644 index 000000000..819ee4cfb --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Execution/FileCopierTest.cs @@ -0,0 +1,175 @@ +using Microservices.FileCopier.Execution; +using Moq; +using NUnit.Framework; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Messaging; +using Smi.Common.Options; +using Smi.Common.Tests; +using System; +using System.IO.Abstractions.TestingHelpers; + +namespace Microservices.FileCopier.Tests.Execution +{ + public class FileCopierTest + { + private FileCopierOptions _options; + + private MockFileSystem _mockFileSystem; + private const string FileSystemRoot = "smi"; + private string _relativeSrc; + private readonly byte[] _expectedContents = { 0b00, 0b01, 0b10, 0b11 }; + private ExtractFileMessage _requestMessage; + + #region Fixture Methods + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + + _options = new FileCopierOptions + { + NoVerifyRoutingKey = "noverify", + }; + + _mockFileSystem = new MockFileSystem(); + _mockFileSystem.Directory.CreateDirectory(FileSystemRoot); + _relativeSrc = _mockFileSystem.Path.Combine("input", "a.dcm"); + string src = _mockFileSystem.Path.Combine("smi", _relativeSrc); + _mockFileSystem.Directory.CreateDirectory(_mockFileSystem.Directory.GetParent(src).FullName); + _mockFileSystem.File.WriteAllBytes(src, _expectedContents); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() + { + _requestMessage = new ExtractFileMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "123", + ExtractionDirectory = "extract", + DicomFilePath = _relativeSrc, + OutputPath = "out.dcm", + }; + } + + [TearDown] + public void TearDown() { } + + #endregion + + #region Tests + + [Test] + public void Test_FileCopier_HappyPath() + { + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractedFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractedFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + var requestHeader = new MessageHeader(); + + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractedFileStatus.Copied, + OutputFilePath = _requestMessage.OutputPath, + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); + + string expectedDest = _mockFileSystem.Path.Combine("smi", "extract", "out.dcm"); + Assert.True(_mockFileSystem.File.Exists(expectedDest)); + Assert.AreEqual(_expectedContents, _mockFileSystem.File.ReadAllBytes(expectedDest)); + } + + [Test] + public void Test_FileCopier_MissingFile_SendsMessage() + { + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractedFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractedFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + _requestMessage.DicomFilePath = "missing.dcm"; + var requestHeader = new MessageHeader(); + + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractedFileStatus.FileMissing, + OutputFilePath = null, + StatusMessage = $"Could not find '{_mockFileSystem.Path.Combine(FileSystemRoot, "missing.dcm")}'" + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); + } + + [Test] + public void Test_FileCopier_ExistingOutputFile_IsOverwritten() + { + var mockProducerModel = new Mock(MockBehavior.Strict); + ExtractedFileStatusMessage sentStatusMessage = null; + string sentRoutingKey = null; + mockProducerModel + .Setup(x => x.SendMessage(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((IMessage message, IMessageHeader header, string routingKey) => + { + sentStatusMessage = (ExtractedFileStatusMessage)message; + sentRoutingKey = routingKey; + }) + .Returns(() => null); + + var requestHeader = new MessageHeader(); + string expectedDest = _mockFileSystem.Path.Combine("smi", "extract", "out.dcm"); + _mockFileSystem.Directory.GetParent(expectedDest).Create(); + _mockFileSystem.File.WriteAllBytes(expectedDest, new byte[] { 0b0 }); + + var copier = new ExtractionFileCopier(_options, mockProducerModel.Object, FileSystemRoot, _mockFileSystem); + copier.ProcessMessage(_requestMessage, requestHeader); + + var expectedStatusMessage = new ExtractedFileStatusMessage(_requestMessage) + { + DicomFilePath = _requestMessage.DicomFilePath, + Status = ExtractedFileStatus.Copied, + OutputFilePath = _requestMessage.OutputPath, + StatusMessage = null, + }; + Assert.AreEqual(expectedStatusMessage, sentStatusMessage); + Assert.AreEqual(_options.NoVerifyRoutingKey, sentRoutingKey); + Assert.AreEqual(_expectedContents, _mockFileSystem.File.ReadAllBytes(expectedDest)); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs new file mode 100644 index 000000000..d91798d75 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Messaging/FileCopyQueueConsumerTest.cs @@ -0,0 +1,158 @@ +using Microservices.FileCopier.Execution; +using Microservices.FileCopier.Messaging; +using Moq; +using Newtonsoft.Json; +using NUnit.Framework; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Framing; +using Smi.Common.Messages; +using Smi.Common.Messages.Extraction; +using Smi.Common.Tests; +using System; +using System.Collections.Generic; +using System.Text; + + +namespace Microservices.FileCopier.Tests.Messaging +{ + public class FileCopyQueueConsumerTest + { + #region Fixture Methods + + private ExtractFileMessage _message; + private Mock _mockModel; + private Mock _mockFileCopier; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() { } + + #endregion + + #region Test Methods + + [SetUp] + public void SetUp() + { + _message = new ExtractFileMessage + { + JobSubmittedAt = DateTime.UtcNow, + ExtractionJobIdentifier = Guid.NewGuid(), + ProjectNumber = "1234", + ExtractionDirectory = "foo", + DicomFilePath = "foo.dcm", + IsIdentifiableExtraction = true, + OutputPath = "bar", + }; + _mockModel = new Mock(MockBehavior.Strict); + _mockModel.Setup(x => x.IsClosed).Returns(false); + _mockModel.Setup(x => x.BasicAck(It.IsAny(), It.IsAny())); + _mockModel.Setup(x => x.BasicNack(It.IsAny(), It.IsAny(), It.IsAny())); + + _mockFileCopier = new Mock(MockBehavior.Strict); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())); + } + + [TearDown] + public void TearDown() { } + + private static BasicDeliverEventArgs GetMockDeliverArgs(ExtractFileMessage message) + { + var mockDeliverArgs = Mock.Of(MockBehavior.Strict); + mockDeliverArgs.DeliveryTag = 1; + mockDeliverArgs.Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(message)); + mockDeliverArgs.BasicProperties = new BasicProperties { Headers = new Dictionary() }; + var header = new MessageHeader(); + header.Populate(mockDeliverArgs.BasicProperties.Headers); + // Have to convert these to bytes since RabbitMQ normally does that when sending + mockDeliverArgs.BasicProperties.Headers["MessageGuid"] = Encoding.UTF8.GetBytes(header.MessageGuid.ToString()); + mockDeliverArgs.BasicProperties.Headers["ProducerExecutableName"] = Encoding.UTF8.GetBytes(header.ProducerExecutableName); + mockDeliverArgs.BasicProperties.Headers["Parents"] = Encoding.UTF8.GetBytes(string.Join("->", header.Parents)); + return mockDeliverArgs; + } + + #endregion + + #region Tests + + [Test] + public void Test_FileCopyQueueConsumer_ValidMessage_IsAcked() + { + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + consumer.ProcessMessage(mockDeliverArgs); + + new TestTimelineAwaiter().Await(() => consumer.AckCount == 1 && consumer.NackCount == 0); + } + + [Test] + public void Test_FileCopyQueueConsumer_ApplicationException_IsNacked() + { + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + consumer.ProcessMessage(mockDeliverArgs); + + new TestTimelineAwaiter().Await(() => consumer.AckCount == 0 && consumer.NackCount == 1); + } + + [Test] + public void Test_FileCopyQueueConsumer_UnknownException_CallsFatalCallback() + { + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + consumer.OnFatal += (sender, _) => fatalCalled = true; + + consumer.ProcessMessage(mockDeliverArgs); + + new TestTimelineAwaiter().Await(() => fatalCalled, "Expected Fatal to be called"); + Assert.AreEqual(0, consumer.AckCount); + Assert.AreEqual(0, consumer.NackCount); + } + + [Test] + public void Test_FileCopyQueueConsumer_AnonExtraction_ThrowsException() + { + _message.IsIdentifiableExtraction = false; + BasicDeliverEventArgs mockDeliverArgs = GetMockDeliverArgs(_message); + + _mockFileCopier.Reset(); + _mockFileCopier.Setup(x => x.ProcessMessage(It.IsAny(), It.IsAny())).Throws(); + + var consumer = new FileCopyQueueConsumer(_mockFileCopier.Object); + consumer.SetModel(_mockModel.Object); + + var fatalCalled = false; + consumer.OnFatal += (sender, _) => fatalCalled = true; + + consumer.ProcessMessage(mockDeliverArgs); + + new TestTimelineAwaiter().Await(() => fatalCalled, "Expected Fatal to be called"); + Assert.AreEqual(0, consumer.AckCount); + Assert.AreEqual(0, consumer.NackCount); + } + + #endregion + } +} diff --git a/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj new file mode 100644 index 000000000..3ea4d2464 --- /dev/null +++ b/tests/microservices/Microservices.FileCopier.Tests/Microservices.FileCopier.Tests.csproj @@ -0,0 +1,28 @@ + + + Microservices.FileCopier.Tests + netcoreapp3.1 + false + bin\$(Platform)\$(Configuration)\ + x64 + false + false + true + 8.0 + full + true + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs b/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs index 7cde5e278..52ae6a455 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/IdentifierMapperTests.cs @@ -106,7 +106,7 @@ public void TestIdentifierSwap_NoCache(DatabaseType type, Test test) public void TestIdentifierSwap_RegexVsDeserialize(DatabaseType type, int batchSize, int numberOfRandomTagsPerDicom) { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var mappingDataTable = new DataTable("IdMap"); mappingDataTable.Columns.Add("priv"); @@ -535,7 +535,7 @@ public void TestSwapCache() DiscoveredDatabase db = GetCleanedServer(DatabaseType.MicrosoftSQLServer); - GlobalOptions options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IdentifierMapperOptions = new IdentifierMapperOptions { MappingConnectionString = db.Server.Builder.ConnectionString, diff --git a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj index cb30ae879..3d1fad127 100644 --- a/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj +++ b/tests/microservices/Microservices.IdentifierMapper.Tests/Microservices.IdentifierMapper.Tests.csproj @@ -13,8 +13,8 @@ true - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs index 9b5abefbe..a30079c8b 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/IsIdentifiable_TestRegexDetections.cs @@ -21,8 +21,60 @@ public void TestChiInString() Assert.AreEqual("0101010101", p.Word); Assert.AreEqual(10, p.Offset); + } + [Test] + public void TestCaching() + { + var runner = new TestRunner("hey there,0101010101 excited to see you"); + runner.Run(); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(1,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(2,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); + + runner.ValueToTest = "ffffff"; + runner.Run(); + Assert.AreEqual(2,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(3,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(4,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); + + runner.FieldToTest = "OtherField"; + runner.Run(); + Assert.AreEqual(4,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(5,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(6,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); } + [Test] + public void Test_NoCaching() + { + var runner = new TestRunner("hey there,0101010101 excited to see you"); + runner.MaxValidationCacheSize = 0; + runner.Run(); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(1,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(2,runner.ValidateCacheMisses); + runner.Run(); + Assert.AreEqual(0,runner.ValidateCacheHits); + Assert.AreEqual(3,runner.ValidateCacheMisses); + runner.Run(); + } [TestCase("DD3 7LB")] [TestCase("dd3 7lb")] [TestCase("dd37lb")] @@ -256,27 +308,28 @@ public void TestSopDoesNotMatch() private class TestRunner : IsIdentifiableAbstractRunner { - private readonly string _fieldToTest; - private readonly string _valueToTest; + public string FieldToTest {get;set; } + public string ValueToTest {get;set; } public readonly List ResultsOfValidate = new List(); public TestRunner(string valueToTest) : base(new TestOpts()) { - _valueToTest = valueToTest; + ValueToTest = valueToTest; + FieldToTest = "field"; } public TestRunner(string valueToTest, TestOpts opts, string fieldToTest = "field") : base(opts) { - _fieldToTest = fieldToTest; - _valueToTest = valueToTest; + FieldToTest = fieldToTest; + ValueToTest = valueToTest; } public override int Run() { - ResultsOfValidate.AddRange(Validate(_fieldToTest, _valueToTest).OrderBy(v => v.Offset)); + ResultsOfValidate.AddRange(Validate(FieldToTest, ValueToTest).OrderBy(v => v.Offset)); CloseReports(); return 0; } diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj index 69b724b3b..da4b5766f 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/Microservices.IsIdentifiable.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs index af3e287c2..f46724335 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ReviewerTests/SymbolsRulesFactoryTests.cs @@ -1,4 +1,5 @@ -using IsIdentifiableReviewer.Out; +using System; +using IsIdentifiableReviewer.Out; using Microservices.IsIdentifiable.Failures; using Microservices.IsIdentifiable.Reporting; using NUnit.Framework; @@ -7,20 +8,67 @@ namespace Microservices.IsIdentifiable.Tests.ReviewerTests { public class SymbolsRulesFactoryTests { - - [TestCase("12/34/56",@"^\d\d/\d\d/\d\d$")] - [TestCase("1 6",@"^\d\ \d$")] + [TestCase("MR Head 12-11-20","12-11-20",@"(\d\d-\d\d-\d\d)$",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 12/34/56","12/34/56",@"(\d\d/\d\d/\d\d)$",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-[A-Z][A-Z][A-Z]-[a-z]\d)",SymbolsRuleFactoryMode.Full)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(123-[A-Z][A-Z][A-Z]-[a-z]4)",SymbolsRuleFactoryMode.CharactersOnly)] + [TestCase("CT Head - 123-ABC-n4 fishfish","123-ABC-n4",@"(\d\d\d-ABC-n\d)",SymbolsRuleFactoryMode.DigitsOnly)] + [TestCase("123","123",@"^(\d\d\d)$",SymbolsRuleFactoryMode.Full)] + public void TestSymbols_OnePart(string input,string part, string expectedOutput,SymbolsRuleFactoryMode mode) + { + var f = new SymbolsRulesFactory(){Mode = mode}; + + var failure = new Failure(new[] {new FailurePart(part, FailureClassification.Person, input.IndexOf(part))}) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + + + [TestCase("12 Morton Street","12","eet",@"^(\d\d).*([a-z][a-z][a-z])$")] + [TestCase("Morton MR Smith","MR","Smith",@"([A-Z][A-Z]).*([A-Z][a-z][a-z][a-z][a-z])$")] + public void TestSymbols_TwoParts_NoOverlap(string input,string part1,string part2, string expectedOutput) + { + var f = new SymbolsRulesFactory(); - [TestCase("abc\n123",@"^[a-z][a-z][a-z]\n\d\d\d$")] - public void TestSymbols(string input, string expectedOutput) + var failure = new Failure(new[] + { + new FailurePart(part1, FailureClassification.Person, input.IndexOf(part1)), + new FailurePart(part2, FailureClassification.Person, input.IndexOf(part2)) + }) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + + [TestCase("Clowns","Cl","lowns",@"^[A-Z][a-z][a-z][a-z][a-z][a-z]$")] + public void TestSymbols_TwoParts_Overlap(string input,string part1,string part2, string expectedOutput) + { + var f = new SymbolsRulesFactory(); + + var failure = new Failure(new[] + { + new FailurePart(part1, FailureClassification.Person, input.IndexOf(part1)), + new FailurePart(part2, FailureClassification.Person, input.IndexOf(part2)) + }) + { + ProblemValue = input + }; + + Assert.AreEqual(expectedOutput,f.GetPattern(this, failure)); + } + [Test] + public void TestNoParts() { var f = new SymbolsRulesFactory(); + var ex = Assert.Throws(()=> f.GetPattern(this, new Failure(new FailurePart[0]) {ProblemValue = "fdslkfl;asdf"})); + Assert.AreEqual("Failure had no Parts",ex.Message); - Assert.AreEqual( - expectedOutput, - f.GetPattern(this, new Failure(new FailurePart[0]) {ProblemValue = input}) - ); } } } \ No newline at end of file diff --git a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs index 241c3cb28..03417f0d1 100644 --- a/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs +++ b/tests/microservices/Microservices.IsIdentifiable.Tests/ServiceTests/IsIdentifiableHostTests.cs @@ -25,7 +25,7 @@ public void OneTimeSetUp() [Test] public void TestClassifierName_NoClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IsIdentifiableOptions.ClassifierType = ""; var ex = Assert.Throws(() => new IsIdentifiableHost(options, false)); @@ -35,7 +35,7 @@ public void TestClassifierName_NoClassifier() [Test] public void TestClassifierName_NotRecognized() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.WorkDirectory; options.IsIdentifiableOptions.ClassifierType = "HappyFunTimes"; @@ -46,13 +46,15 @@ public void TestClassifierName_NotRecognized() [Test] public void TestClassifierName_ValidClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); var testDcm = new FileInfo(Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm")); Path.Combine(TestContext.CurrentContext.TestDirectory, nameof(TestClassifierName_ValidClassifier), "f1.dcm"); TestData.Create(testDcm); using (var tester = new MicroserviceTester(options.RabbitOptions, options.IsIdentifiableOptions)) { + tester.CreateExchange(options.IsIdentifiableOptions.IsIdentifiableProducerOptions.ExchangeName, null); + options.IsIdentifiableOptions.ClassifierType = typeof(RejectAllClassifier).FullName; options.IsIdentifiableOptions.DataDirectory = TestContext.CurrentContext.TestDirectory; @@ -60,14 +62,14 @@ public void TestClassifierName_ValidClassifier() Assert.IsNotNull(host); host.Start(); - tester.SendMessage(options.IsIdentifiableOptions, new ExtractFileStatusMessage() + tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage() { DicomFilePath = "yay.dcm", - AnonymisedFileName = testDcm.FullName, + OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", - Status = ExtractFileStatus.Anonymised + Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); @@ -78,7 +80,7 @@ public void TestClassifierName_ValidClassifier() [Test] public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() { - var options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + var options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); // Create a test data directory containing IsIdentifiableRules with 0 rules, and tessdata with the eng.traineddata classifier // TODO(rkm 2020-04-14) This is a stop-gap solution until the tests are properly refactored @@ -103,14 +105,14 @@ public void TestIsIdentifiable_TesseractStanfordDicomFileClassifier() var host = new IsIdentifiableHost(options, false); host.Start(); - tester.SendMessage(options.IsIdentifiableOptions, new ExtractFileStatusMessage + tester.SendMessage(options.IsIdentifiableOptions, new ExtractedFileStatusMessage { DicomFilePath = "yay.dcm", - AnonymisedFileName = testDcm.FullName, + OutputFilePath = testDcm.FullName, ProjectNumber = "100", ExtractionDirectory = "./fish", StatusMessage = "yay!", - Status = ExtractFileStatus.Anonymised + Status = ExtractedFileStatus.Anonymised }); var awaiter = new TestTimelineAwaiter(); diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs b/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs index 6a82b7b12..ea6afbd5e 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Execution/Processing/ImageMessageProcessorTests_NoMongo.cs @@ -31,7 +31,7 @@ public void OneTimeSetUp() [SetUp] public void SetUp() { - _testOptions = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + _testOptions = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); } /// diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj index 53807b78a..fbc233326 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/Microservices.MongoDBPopulator.Tests.csproj @@ -13,7 +13,7 @@ true - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs b/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs index fe13a03a0..4413646bd 100644 --- a/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs +++ b/tests/microservices/Microservices.MongoDBPopulator.Tests/MongoDbPopulatorTestHelper.cs @@ -78,7 +78,7 @@ public void SetupSuite() public static GlobalOptions GetNewMongoDbPopulatorOptions() { - GlobalOptions options = GlobalOptions.Load("default.yaml", TestContext.CurrentContext.TestDirectory); + GlobalOptions options = new GlobalOptionsFactory().Load("default.yaml", TestContext.CurrentContext.TestDirectory); options.MongoDatabases.DicomStoreOptions.DatabaseName = TestDbName; options.MongoDbPopulatorOptions.MongoDbFlushTime = 1; //1 second