Skip to content

Commit

Permalink
Merge branch 'develop' into dependabot/nuget/develop/NLog-5.0.0
Browse files Browse the repository at this point in the history
  • Loading branch information
jas88 authored May 17, 2022
2 parents 40cbe1b + 05acc3e commit 6e3c099
Show file tree
Hide file tree
Showing 45 changed files with 553 additions and 71 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:

steps:
- name: Install Node for coverage reporting
uses: actions/setup-node@v3.1.1
uses: actions/setup-node@v3.2.0
with:
node-version: '16.x'
- name: LCov merger tool
Expand Down
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

...

### Changed

- 'View Aggregate' now explicitly applies an ORDER BY count descending.
- New CatalogueItems are now always marked Core (affects drag and drop and new Catalogue creation) - [#1165](https://github.com/HicServices/RDMP/issues/1165),[#1164](https://github.com/HicServices/RDMP/issues/1164)

### Added

- Added `CrashAtEnd` system for DLE that allows Attachers to flag a load as a failure without halting execution [#1157](https://github.com/HicServices/RDMP/issues/1157)
- Added UserSettings editing UI to Console Gui

## [7.0.12] - 2022-05-16

### Added
Expand Down
2 changes: 1 addition & 1 deletion Documentation/CodeTutorials/Packages.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.4](https://www.nuget.org/packages/HIC.FansiSql/2.0.4) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer |
| HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [1.0.1](https://www.nuget.org/packages/HIC.BadMedicine/1.0.1) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses |
| SSH.NET | [GitHub](https://github.com/sshnet/SSH.NET) | [2020.0.1](https://www.nuget.org/packages/SSH.NET/2020.0.1) | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers |
| Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.0](https://www.nuget.org/packages/Moq/4.18.0) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt) | Mock objects during unit testing |
| Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.1](https://www.nuget.org/packages/Moq/4.18.1) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt) | Mock objects during unit testing |
| [Nunit](https://nunit.org/) |[GitHub](https://github.com/nunit/nunit) | [3.13.3](https://www.nuget.org/packages/NUnit/3.13.3) | [MIT](https://opensource.org/licenses/MIT) | Unit testing |
| Microsoft.NET.Test.Sdk | [GitHub](https://github.com/microsoft/vstest/) | [17.2.0](https://www.nuget.org/packages/Microsoft.NET.Test.Sdk/17.2.0) | [MIT](https://opensource.org/licenses/MIT) | Required for running tests| |
| NUnit3TestAdapter | [GitHub](https://github.com/nunit/nunit3-vs-adapter)| [3.13.3](https://www.nuget.org/packages/NUnit3TestAdapter/3.13.3) | [MIT](https://opensource.org/licenses/MIT) | Run unit tests from within Visual Studio |
Expand Down
2 changes: 1 addition & 1 deletion Plugins/Plugin.Test/Plugin.Test.nuspec
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
<dependency id="CommandLineParser" version="2.8.0" />
<dependency id="SSH.NET" version="2020.0.1" />
<dependency id="NUnit" version="3.13.3" />
<dependency id="Moq" version="4.18.0" />
<dependency id="Moq" version="4.18.1" />
<dependency id="NLog" version="5.0.0" />
<dependency id="YamlDotNet" version="11.2.1" />
<dependency id="Microsoft.NET.Test.Sdk" version="17.2.0" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public void CohortIdentificationConfiguration_Join_PatientIndexTable()
var importer = new TableInfoImporter(CatalogueRepository,rTbl);
importer.DoImport(out var rTi,out ColumnInfo[] rColInfos);

var fe = new ForwardEngineerCatalogue(rTi,rColInfos,true);
var fe = new ForwardEngineerCatalogue(rTi,rColInfos);
fe.ExecuteForwardEngineering(cata);

//Should now be 1 Catalogue with all the columns (tables will have to be joined to build the query though)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,8 @@ REFERENCES [dbo].[Tests] ([TestId])

importer2.DoImport(out t2, out c2);

var engineer1 = new ForwardEngineerCatalogue(t1, c1, true);
var engineer2 = new ForwardEngineerCatalogue(t2, c2, true);
var engineer1 = new ForwardEngineerCatalogue(t1, c1);
var engineer2 = new ForwardEngineerCatalogue(t2, c2);

engineer1.ExecuteForwardEngineering(out cata1,out cataItems1,out eis1);
engineer2.ExecuteForwardEngineering(out cata2, out cataItems2, out eis2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void Test_DatabaseTypeQueryWithParameter_IntParameter(DatabaseType dbType
var importer = new TableInfoImporter(CatalogueRepository, tbl);
importer.DoImport(out var ti,out var ci);

var engineer = new ForwardEngineerCatalogue(ti, ci,true);
var engineer = new ForwardEngineerCatalogue(ti, ci);
engineer.ExecuteForwardEngineering(out var cata, out var cis, out var ei);
/////////////////////////////////////////////////////////////////////////////////////////////////////////

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,10 @@ public void CreateANOVersion_TestSkippingTables(bool tableInfoAlreadyExistsForSk
fromNeckColumnInfo.Single(c => c.GetRuntimeName().Equals("Vertebrae")), ExtractionJoinType.Inner, null
);

var cataEngineer = new ForwardEngineerCatalogue(fromHeadsTableInfo, fromHeadsColumnInfo, true);
var cataEngineer = new ForwardEngineerCatalogue(fromHeadsTableInfo, fromHeadsColumnInfo);
cataEngineer.ExecuteForwardEngineering(out var cata,out var cataItems,out var extractionInformations);

var cataEngineer2 = new ForwardEngineerCatalogue(fromNeckTableInfo, fromNeckColumnInfo, true);
var cataEngineer2 = new ForwardEngineerCatalogue(fromNeckTableInfo, fromNeckColumnInfo);
cataEngineer2.ExecuteForwardEngineering(cata);

//4 extraction informations in from Catalogue (2 from Heads and 2 from Necks)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public void SynchronizationTests_ColumnAdded(bool acceptChanges)
[TestCase(false)]
public void SynchronizationTests_ColumnAddedWithCatalogue(bool acceptChanges)
{
ForwardEngineerCatalogue cataEngineer = new ForwardEngineerCatalogue(tableInfoCreated,columnInfosCreated,true);
ForwardEngineerCatalogue cataEngineer = new ForwardEngineerCatalogue(tableInfoCreated, columnInfosCreated);
cataEngineer.ExecuteForwardEngineering(out var cata, out var cataItems, out var extractionInformations);

try
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ definitionID int
var importer = new TableValuedFunctionImporter(CatalogueRepository, tblvf);
importer.DoImport(out var tbl,out var cols);

var engineer = new ForwardEngineerCatalogue(tbl, cols, true);
var engineer = new ForwardEngineerCatalogue(tbl, cols);
engineer.ExecuteForwardEngineering(out var cata, out var cis, out var eis);

Assert.AreEqual("chi", eis[0].GetRuntimeName());
Expand Down Expand Up @@ -238,7 +238,7 @@ private void CreateANormalCatalogue()

importer.DoImport(out var tbl,out var cols);

var engineer = new ForwardEngineerCatalogue(tbl, cols, true);
var engineer = new ForwardEngineerCatalogue(tbl, cols);
engineer.ExecuteForwardEngineering(out var cata,out var cis, out var eis);

_nonTvfExtractionIdentifier = eis.Single();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ protected ITableInfo Import(DiscoveredTable tbl, LoadMetadata lmd, LogManager lo
importer.DoImport(out var ti, out var cis);

//create Catalogue
var forwardEngineer = new ForwardEngineerCatalogue(ti, cis, true);
var forwardEngineer = new ForwardEngineerCatalogue(ti, cis);
forwardEngineer.ExecuteForwardEngineering(out var cata, out var cataItems, out var eis);

//make the catalogue use the load configuration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ namespace Rdmp.Core.Tests.DataLoad.Engine.Integration
{
public class ToMemoryDataLoadJob : ToMemoryDataLoadEventListener, IDataLoadJob
{
private List<NotifyEventArgs> _crashAtEnd = new ();
/// <inheritdoc/>
public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();

public ToMemoryDataLoadJob(bool throwOnErrorEvents = true): base(throwOnErrorEvents)
{
}
Expand Down Expand Up @@ -64,5 +68,10 @@ public ColumnInfo[] GetAllColumns()
{
return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
}
/// <inheritdoc/>
public void CrashAtEnd(NotifyEventArgs because)
{
_crashAtEnd.Add(because);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public override void Execute()
var c = _catalogue;
var existingColumnInfos = _existingColumnInfos;

if(c == null)
var repo = BasicActivator.RepositoryLocator.CatalogueRepository;

if (c == null)
{
if (BasicActivator.SelectObject(new DialogArgs() {
WindowTitle = "Add CatalogueItem",
Expand Down Expand Up @@ -102,7 +104,11 @@ public override void Execute()

//set the associated column if they did pick it
if(columnInfo != null)
{
ci.SetColumnInfo(columnInfo);
// also make extractable
new ExtractionInformation(repo, ci, columnInfo, columnInfo.GetFullyQualifiedName());
}

ci.SaveToDatabase();

Expand All @@ -117,9 +123,12 @@ public override void Execute()
if(AlreadyInCatalogue(columnInfo, existingColumnInfos))
continue;

var ci = new CatalogueItem(BasicActivator.RepositoryLocator.CatalogueRepository, c, columnInfo.GetRuntimeName());
var ci = new CatalogueItem(repo, c, columnInfo.GetRuntimeName());
ci.SetColumnInfo(columnInfo);
ci.SaveToDatabase();

// also make extractable
new ExtractionInformation(repo, ci, columnInfo, columnInfo.GetFullyQualifiedName());
}

Publish(c);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public override void Execute()

if (_createCatalogue)
{
var forwardEngineer = new ForwardEngineerCatalogue(ti, cis, true);
var forwardEngineer = new ForwardEngineerCatalogue(ti, cis);
forwardEngineer.ExecuteForwardEngineering(out c, out _, out _);

BasicActivator.Show($"Successfully imported new Catalogue { c.Name} with ID {c.ID}");
Expand Down
2 changes: 1 addition & 1 deletion Rdmp.Core/CommandExecution/BasicActivateItems.cs
Original file line number Diff line number Diff line change
Expand Up @@ -727,7 +727,7 @@ public virtual CohortCreationRequest GetCohortCreationRequest(ExternalCohortTabl
public virtual ICatalogue CreateAndConfigureCatalogue(ITableInfo tableInfo, ColumnInfo[] extractionIdentifierColumns, string initialDescription, IProject projectSpecific, string catalogueFolder)
{
// Create a new Catalogue based on the table info
var engineer = new ForwardEngineerCatalogue(tableInfo,tableInfo.ColumnInfos,true);
var engineer = new ForwardEngineerCatalogue(tableInfo,tableInfo.ColumnInfos);
engineer.ExecuteForwardEngineering(out ICatalogue cata, out _, out ExtractionInformation[] eis);

// if we know the linkable private identifier column(s)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -581,7 +581,7 @@ private ICatalogue ImportCatalogue(DiscoveredTable tbl)
}
private ICatalogue ImportCatalogue(ITableInfo ti)
{
var forwardEngineer = new ForwardEngineerCatalogue(ti,ti.ColumnInfos,true);
var forwardEngineer = new ForwardEngineerCatalogue(ti,ti.ColumnInfos);
forwardEngineer.ExecuteForwardEngineering(out var cata, out _,out ExtractionInformation[] eis);

//get descriptions of the columns from BadMedicine
Expand Down
23 changes: 6 additions & 17 deletions Rdmp.Core/Curation/ForwardEngineerCatalogue.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,16 @@ public class ForwardEngineerCatalogue
{
private readonly ITableInfo _tableInfo;
private readonly ColumnInfo[] _columnInfos;
private readonly bool _markAllExtractable;

/// <summary>
/// Sets up the class to create a new <see cref="Catalogue"/> from the supplied table reference
/// </summary>
/// <param name="tableInfo"></param>
/// <param name="columnInfos"></param>
/// <param name="markAllExtractable"></param>
public ForwardEngineerCatalogue(ITableInfo tableInfo, ColumnInfo[] columnInfos, bool markAllExtractable = false)
public ForwardEngineerCatalogue(ITableInfo tableInfo, ColumnInfo[] columnInfos)
{
_tableInfo = tableInfo;
_columnInfos = columnInfos;
_markAllExtractable = markAllExtractable;
}


Expand Down Expand Up @@ -84,19 +81,11 @@ public void ExecuteForwardEngineering(ICatalogue intoExistingCatalogue,out ICata
//create it with the same name
CatalogueItem cataItem = new CatalogueItem(repo, intoExistingCatalogue, col.Name.Substring(col.Name.LastIndexOf(".") + 1).Trim('[', ']', '`','"'));
catalogueItemsCreated.Add(cataItem);

if (_markAllExtractable)
{
var newExtractionInfo = new ExtractionInformation(repo, cataItem, col, col.Name);
newExtractionInfo.Order = order;
newExtractionInfo.SaveToDatabase();
extractionInformationsCreated.Add(newExtractionInfo);
}
else
{
cataItem.ColumnInfo_ID = col.ID;
cataItem.SaveToDatabase();
}

var newExtractionInfo = new ExtractionInformation(repo, cataItem, col, col.Name);
newExtractionInfo.Order = order;
newExtractionInfo.SaveToDatabase();
extractionInformationsCreated.Add(newExtractionInfo);
}

extractionInformations = extractionInformationsCreated.ToArray();
Expand Down
2 changes: 1 addition & 1 deletion Rdmp.Core/Curation/TableInfoSynchronizer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ private void ForwardEngineerExtractionInformationIfAppropriate(List<ColumnInfo>
"Also make columns Extractable?")))
{
//Create CatalogueItems for the new columns
ForwardEngineerCatalogue c = new ForwardEngineerCatalogue(_tableToSync, added.ToArray(), true);
ForwardEngineerCatalogue c = new ForwardEngineerCatalogue(_tableToSync, added.ToArray());

//In the Catalogue
c.ExecuteForwardEngineering(relatedCatalogues[0],out var cata,out var cis, out var eis);
Expand Down
9 changes: 9 additions & 0 deletions Rdmp.Core/DataLoad/Engine/Job/DataLoadJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ public class DataLoadJob : IDataLoadJob
public HICDatabaseConfiguration Configuration { get; set; }
public object Payload { get; set; }

private List<NotifyEventArgs> _crashAtEnd = new();

public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();


private string _loggingTask;

Expand Down Expand Up @@ -202,5 +206,10 @@ public ColumnInfo[] GetAllColumns()
{
return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
}

public void CrashAtEnd(NotifyEventArgs because)
{
_crashAtEnd.Add(because);
}
}
}
18 changes: 17 additions & 1 deletion Rdmp.Core/DataLoad/Engine/Job/IDataLoadJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.

using System;
using System.Collections.Generic;
using Rdmp.Core.Curation;
using Rdmp.Core.Curation.Data;
Expand Down Expand Up @@ -35,9 +36,15 @@ public interface IDataLoadJob : IDataLoadEventListener, IDisposeAfterDataLoad
/// </summary>
object Payload { get; set; }

/// <summary>
/// Collection of all calls to <see cref="CrashAtEnd"/>. If there are any
/// of these at the end of the load they will be notified and a crash exit code will be
/// returned (but otherwise the load will complete normally).
/// </summary>
IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages { get; }
List<ITableInfo> RegularTablesToLoad { get; }
List<ITableInfo> LookupTablesToLoad { get; }

IRDMPPlatformRepositoryServiceLocator RepositoryLocator { get; }

void StartLogging();
Expand All @@ -60,5 +67,14 @@ public interface IDataLoadJob : IDataLoadEventListener, IDisposeAfterDataLoad
/// </summary>
/// <returns></returns>
ColumnInfo[] GetAllColumns();

/// <summary>
/// Call you see that something has gone horribly wrong but want to keep going
/// with the load anyway. Once the load has been completed these will crash
/// the process and result in a non 0 exit code. Archiving and postload operations
/// will still occur.
/// </summary>
/// <param name="because"></param>
void CrashAtEnd(NotifyEventArgs because);
}
}
12 changes: 12 additions & 0 deletions Rdmp.Core/DataLoad/Engine/Job/ThrowImmediatelyDataLoadJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ public void CloseLogging()

public object Payload { get; set; }

private List<NotifyEventArgs> _crashAtEnd = new();

/// <inheritdoc/>
public IReadOnlyCollection<NotifyEventArgs> CrashAtEndMessages => _crashAtEnd.AsReadOnly();


public void AddForDisposalAfterCompletion(IDisposeAfterDataLoad disposable)
{
}
Expand Down Expand Up @@ -99,5 +105,11 @@ public ColumnInfo[] GetAllColumns()
{
return RegularTablesToLoad.SelectMany(t=>t.ColumnInfos).Union(LookupTablesToLoad.SelectMany(t=>t.ColumnInfos)).Distinct().ToArray();
}

/// <inheritdoc/>
public void CrashAtEnd(NotifyEventArgs because)
{
_crashAtEnd.Add(because);
}
}
}
14 changes: 14 additions & 0 deletions Rdmp.Core/DataLoad/Engine/LoadExecution/SingleJobExecution.cs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,20 @@ public ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken cancellation

job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "Completed job " + job.JobID));

if(job.CrashAtEndMessages.Count > 0)
{
job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"There were {job.CrashAtEndMessages.Count} {nameof(IDataLoadJob.CrashAtEndMessages)} registered for job " + job.JobID));

// pop the messages into the handler
foreach (var m in job.CrashAtEndMessages)
{
job.OnNotify(job, m); // depending on the listener these may break flow of control (e.g.
}

// return failed (even if the messages are all warnings)
return ExitCodeType.Error;
}

return ExitCodeType.Success;
}
catch (OperationCanceledException)
Expand Down
Loading

0 comments on commit 6e3c099

Please sign in to comment.