diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8229d15494..a25cf06a98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- 'View Aggregate' now explicitly applies an ORDER BY count descending.
- New CatalogueItems are now always marked Core (affects drag and drop and new Catalogue creation) - [#1165](https://github.com/HicServices/RDMP/issues/1165),[#1164](https://github.com/HicServices/RDMP/issues/1164)
+- If a Catalogue is defined for a Lookup TableInfo then only Core extractable columns will be released (previously all columns were released) [#692](https://github.com/HicServices/RDMP/issues/692)
### Added
diff --git a/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs b/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs
new file mode 100644
index 0000000000..9991e29185
--- /dev/null
+++ b/Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs
@@ -0,0 +1,57 @@
+// Copyright (c) The University of Dundee 2018-2019
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see .
+
+using NUnit.Framework;
+using Rdmp.Core.Curation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.DataExport.DataExtraction.UserPicks;
+using Tests.Common;
+
+namespace Rdmp.Core.Tests.Curation.Integration
+{
+ public class BundledLookupTableTests : UnitTests
+ {
+ [Test]
+ public void TestLookupGetDataTableFetchSql()
+ {
+ var l = WhenIHaveA();
+ var t =l.PrimaryKey.TableInfo;
+
+ var bundle = new BundledLookupTable(t);
+ Assert.AreEqual("select * from [MyDb]..[ChildTable]", bundle.GetDataTableFetchSql());
+ }
+ [Test]
+ public void TestLookupGetDataTableFetchSql_WithCatalogue()
+ {
+ var l = WhenIHaveA();
+ var t = l.PrimaryKey.TableInfo;
+
+ var engineer = new ForwardEngineerCatalogue(t, t.ColumnInfos);
+ engineer.ExecuteForwardEngineering(out var cata,out _, out var eis);
+
+ var bundle = new BundledLookupTable(t);
+ Assert.AreEqual(@"
+SELECT
+
+ChildCol,
+Desc
+FROM
+ChildTable", bundle.GetDataTableFetchSql());
+
+
+ // ei 1 is suplemental now
+ eis[1].ExtractionCategory = ExtractionCategory.Supplemental;
+ eis[1].SaveToDatabase();
+
+ Assert.AreEqual(@"
+SELECT
+
+ChildCol
+FROM
+ChildTable", bundle.GetDataTableFetchSql());
+ }
+ }
+}
diff --git a/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs b/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs
index 4caec309c8..76eb8bfa99 100644
--- a/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs
+++ b/Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs
@@ -10,6 +10,7 @@
using System.Linq;
using FAnsi.Discovery;
using Rdmp.Core.DataExport.DataExtraction.FileOutputFormats;
+using Rdmp.Core.DataExport.DataExtraction.UserPicks;
using Rdmp.Core.DataViewing;
using ReusableLibraryCode.DataAccess;
diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs
index 6455217b04..cd46a23d82 100644
--- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs
+++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs
@@ -471,8 +471,7 @@ protected override void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlT
protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, DirectoryInfo lookupDir,
IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription)
{
- var tbl = lookup.TableInfo.Discover(DataAccessContext.DataExport);
- var dt = tbl.GetDataTable();
+ var dt = lookup.GetDataTable();
dt.TableName = GetTableName(_destinationDatabase.Server.GetQuerySyntaxHelper().GetSensibleEntityNameFromString(lookup.TableInfo.Name));
@@ -490,6 +489,8 @@ protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, Dir
}
destinationDb.CreateTable(dt.TableName, dt);
+
+ dt.Dispose();
}
private DiscoveredDatabase GetDestinationDatabase(IDataLoadEventListener listener)
diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs
index a14fe642fa..6a4663ac3f 100644
--- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs
+++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs
@@ -473,9 +473,14 @@ protected virtual void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlTa
}
protected virtual void TryExtractLookupTableImpl( BundledLookupTable lookup, DirectoryInfo lookupDir, IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription)
- {
- //extracts all of them
- var extractTableVerbatim = new ExtractTableVerbatim(lookupDir, _request.Configuration.Separator, DateFormat,lookup.TableInfo.Discover(DataAccessContext.DataExport));
+ {
+ //extract the lookup table SQL
+ var sql = lookup.GetDataTableFetchSql();
+
+ var extractTableVerbatim = new ExtractTableVerbatim(
+ lookup.TableInfo.Discover(DataAccessContext.DataExport).Database.Server,
+ sql,lookup.TableInfo.GetRuntimeName(), lookupDir, _request.Configuration.Separator, DateFormat);
+
linesWritten = extractTableVerbatim.DoExtraction();
destinationDescription = extractTableVerbatim.OutputFilename;
}
diff --git a/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs b/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs
index 61aa07a615..df824dcde9 100644
--- a/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs
+++ b/Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs
@@ -5,7 +5,10 @@
// You should have received a copy of the GNU General Public License along with RDMP. If not, see .
using System;
+using System.Data;
using Rdmp.Core.Curation.Data;
+using Rdmp.Core.QueryBuilding;
+using ReusableLibraryCode.DataAccess;
namespace Rdmp.Core.DataExport.DataExtraction.UserPicks
{
@@ -29,5 +32,50 @@ public override string ToString()
{
return TableInfo.ToString();
}
+
+ ///
+ /// Reads lookup data from the using
+ ///
+ ///
+ public DataTable GetDataTable()
+ {
+ var tbl = TableInfo.Discover(DataAccessContext.DataExport);
+ var server = tbl.Database.Server;
+
+ var dt = new DataTable();
+
+ using (var con = server.GetConnection())
+ {
+ con.Open();
+ using (var da = server.GetDataAdapter(
+ server.GetCommand(GetDataTableFetchSql(),con)))
+ {
+ da.Fill(dt);
+ }
+ }
+ return dt;
+ }
+
+ public string GetDataTableFetchSql()
+ {
+ var catas = TableInfo.GetAllRelatedCatalogues();
+ QueryBuilder qb;
+
+ if (catas.Length == 1)
+ {
+ // if there is a Catalogue associated with this TableInfo use its extraction instead
+ var cata = catas[0];
+ var eis = cata.GetAllExtractionInformation(ExtractionCategory.Core);
+
+ if(eis.Length > 0)
+ {
+ qb = new QueryBuilder(null, null, new[] { TableInfo });
+ qb.AddColumnRange(eis);
+ return qb.SQL;
+ }
+ }
+
+ return $"select * from {TableInfo.GetFullyQualifiedName()}";
+ }
}
}
\ No newline at end of file
diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs
index f78a4249b2..a2bf5c5d15 100644
--- a/Tests.Common/UnitTests.cs
+++ b/Tests.Common/UnitTests.cs
@@ -598,11 +598,13 @@ private static void WhenIHaveTwoTables(MemoryDataExportRepository repository,out
{
ti1 = WhenIHaveA(repository);
ti1.Name = "ParentTable";
+ ti1.Database = "MyDb";
ti1.SaveToDatabase();
col1 = new ColumnInfo(repository, "ParentCol", "varchar(10)", ti1);
ti2 = WhenIHaveA(repository);
- ti2.Name = "Child Table";
+ ti2.Name = "ChildTable";
+ ti2.Database = "MyDb";
col2 = new ColumnInfo(repository, "ChildCol", "varchar(10)", ti2);
col3 = new ColumnInfo(repository, "Desc", "varchar(10)", ti2);
}