diff --git a/CHANGELOG.md b/CHANGELOG.md index 8229d15494..a25cf06a98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - 'View Aggregate' now explicitly applies an ORDER BY count descending. - New CatalogueItems are now always marked Core (affects drag and drop and new Catalogue creation) - [#1165](https://github.com/HicServices/RDMP/issues/1165),[#1164](https://github.com/HicServices/RDMP/issues/1164) +- If a Catalogue is defined for a Lookup TableInfo then only Core extractable columns will be released (previously all columns were released) [#692](https://github.com/HicServices/RDMP/issues/692) ### Added diff --git a/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs b/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs new file mode 100644 index 0000000000..9991e29185 --- /dev/null +++ b/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs @@ -0,0 +1,57 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using NUnit.Framework; +using Rdmp.Core.Curation; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.DataExtraction.UserPicks; +using Tests.Common; + +namespace Rdmp.Core.Tests.Curation.Integration +{ + public class BundledLookupTableTests : UnitTests + { + [Test] + public void TestLookupGetDataTableFetchSql() + { + var l = WhenIHaveA(); + var t =l.PrimaryKey.TableInfo; + + var bundle = new BundledLookupTable(t); + Assert.AreEqual("select * from [MyDb]..[ChildTable]", bundle.GetDataTableFetchSql()); + } + [Test] + public void TestLookupGetDataTableFetchSql_WithCatalogue() + { + var l = WhenIHaveA(); + var t = l.PrimaryKey.TableInfo; + + var engineer = new ForwardEngineerCatalogue(t, t.ColumnInfos); + engineer.ExecuteForwardEngineering(out var cata,out _, out var eis); + + var bundle = new BundledLookupTable(t); + Assert.AreEqual(@" +SELECT + +ChildCol, +Desc +FROM +ChildTable", bundle.GetDataTableFetchSql()); + + + // ei 1 is suplemental now + eis[1].ExtractionCategory = ExtractionCategory.Supplemental; + eis[1].SaveToDatabase(); + + Assert.AreEqual(@" +SELECT + +ChildCol +FROM +ChildTable", bundle.GetDataTableFetchSql()); + } + } +} diff --git a/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs b/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs index 4caec309c8..76eb8bfa99 100644 --- a/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs +++ b/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs @@ -10,6 +10,7 @@ using System.Linq; using FAnsi.Discovery; using Rdmp.Core.DataExport.DataExtraction.FileOutputFormats; +using Rdmp.Core.DataExport.DataExtraction.UserPicks; using Rdmp.Core.DataViewing; using ReusableLibraryCode.DataAccess; diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs index 6455217b04..cd46a23d82 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs @@ -471,8 +471,7 @@ protected override void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlT protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, DirectoryInfo lookupDir, IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription) { - var tbl = lookup.TableInfo.Discover(DataAccessContext.DataExport); - var dt = tbl.GetDataTable(); + var dt = lookup.GetDataTable(); dt.TableName = GetTableName(_destinationDatabase.Server.GetQuerySyntaxHelper().GetSensibleEntityNameFromString(lookup.TableInfo.Name)); @@ -490,6 +489,8 @@ protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, Dir } destinationDb.CreateTable(dt.TableName, dt); + + dt.Dispose(); } private DiscoveredDatabase GetDestinationDatabase(IDataLoadEventListener listener) diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs index a14fe642fa..6a4663ac3f 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs @@ -473,9 +473,14 @@ protected virtual void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlTa } protected virtual void TryExtractLookupTableImpl( BundledLookupTable lookup, DirectoryInfo lookupDir, IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription) - { - //extracts all of them - var extractTableVerbatim = new ExtractTableVerbatim(lookupDir, _request.Configuration.Separator, DateFormat,lookup.TableInfo.Discover(DataAccessContext.DataExport)); + { + //extract the lookup table SQL + var sql = lookup.GetDataTableFetchSql(); + + var extractTableVerbatim = new ExtractTableVerbatim( + lookup.TableInfo.Discover(DataAccessContext.DataExport).Database.Server, + sql,lookup.TableInfo.GetRuntimeName(), lookupDir, _request.Configuration.Separator, DateFormat); + linesWritten = extractTableVerbatim.DoExtraction(); destinationDescription = extractTableVerbatim.OutputFilename; } diff --git a/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs b/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs index 61aa07a615..df824dcde9 100644 --- a/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs +++ b/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs @@ -5,7 +5,10 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.Data; using Rdmp.Core.Curation.Data; +using Rdmp.Core.QueryBuilding; +using ReusableLibraryCode.DataAccess; namespace Rdmp.Core.DataExport.DataExtraction.UserPicks { @@ -29,5 +32,50 @@ public override string ToString() { return TableInfo.ToString(); } + + /// + /// Reads lookup data from the using + /// + /// + public DataTable GetDataTable() + { + var tbl = TableInfo.Discover(DataAccessContext.DataExport); + var server = tbl.Database.Server; + + var dt = new DataTable(); + + using (var con = server.GetConnection()) + { + con.Open(); + using (var da = server.GetDataAdapter( + server.GetCommand(GetDataTableFetchSql(),con))) + { + da.Fill(dt); + } + } + return dt; + } + + public string GetDataTableFetchSql() + { + var catas = TableInfo.GetAllRelatedCatalogues(); + QueryBuilder qb; + + if (catas.Length == 1) + { + // if there is a Catalogue associated with this TableInfo use its extraction instead + var cata = catas[0]; + var eis = cata.GetAllExtractionInformation(ExtractionCategory.Core); + + if(eis.Length > 0) + { + qb = new QueryBuilder(null, null, new[] { TableInfo }); + qb.AddColumnRange(eis); + return qb.SQL; + } + } + + return $"select * from {TableInfo.GetFullyQualifiedName()}"; + } } } \ No newline at end of file diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index f78a4249b2..a2bf5c5d15 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -598,11 +598,13 @@ private static void WhenIHaveTwoTables(MemoryDataExportRepository repository,out { ti1 = WhenIHaveA(repository); ti1.Name = "ParentTable"; + ti1.Database = "MyDb"; ti1.SaveToDatabase(); col1 = new ColumnInfo(repository, "ParentCol", "varchar(10)", ti1); ti2 = WhenIHaveA(repository); - ti2.Name = "Child Table"; + ti2.Name = "ChildTable"; + ti2.Database = "MyDb"; col2 = new ColumnInfo(repository, "ChildCol", "varchar(10)", ti2); col3 = new ColumnInfo(repository, "Desc", "varchar(10)", ti2); }