Lookups now check for a single Catalogue and extract only Core column…

…s if so

Fixes #692

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - 'View Aggregate' now explicitly applies an ORDER BY count descending.
 - New CatalogueItems are now always marked Core (affects drag and drop and new Catalogue creation) - [#1165](https://github.com/HicServices/RDMP/issues/1165),[#1164](https://github.com/HicServices/RDMP/issues/1164)
+- If a Catalogue is defined for a Lookup TableInfo then only Core extractable columns will be released (previously all columns were released) [#692](https://github.com/HicServices/RDMP/issues/692)
 
 ### Added
 

Rdmp.Core.Tests/Curation/Integration/BundledLookupTableTests.cs

@@ -0,0 +1,57 @@
+// Copyright (c) The University of Dundee 2018-2019
+// This file is part of the Research Data Management Platform (RDMP).
+// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+// You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
+
+using NUnit.Framework;
+using Rdmp.Core.Curation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.DataExport.DataExtraction.UserPicks;
+using Tests.Common;
+
+namespace Rdmp.Core.Tests.Curation.Integration
+{
+    public class BundledLookupTableTests : UnitTests
+    {
+        [Test]
+        public void TestLookupGetDataTableFetchSql()
+        {
+            var l = WhenIHaveA<Lookup>();
+            var t =l.PrimaryKey.TableInfo;
+
+            var bundle = new BundledLookupTable(t);
+            Assert.AreEqual("select * from [MyDb]..[ChildTable]", bundle.GetDataTableFetchSql());
+        }
+        [Test]
+        public void TestLookupGetDataTableFetchSql_WithCatalogue()
+        {
+            var l = WhenIHaveA<Lookup>();
+            var t = l.PrimaryKey.TableInfo;
+
+            var engineer = new ForwardEngineerCatalogue(t, t.ColumnInfos);
+            engineer.ExecuteForwardEngineering(out var cata,out _, out var eis);
+
+            var bundle = new BundledLookupTable(t);
+            Assert.AreEqual(@"
+SELECT 
+
+ChildCol,
+Desc
+FROM 
+ChildTable", bundle.GetDataTableFetchSql());
+
+
+            // ei 1 is suplemental now
+            eis[1].ExtractionCategory = ExtractionCategory.Supplemental;
+            eis[1].SaveToDatabase();
+
+            Assert.AreEqual(@"
+SELECT 
+
+ChildCol
+FROM 
+ChildTable", bundle.GetDataTableFetchSql());
+        }
+    }
+}

Rdmp.Core/DataExport/DataExtraction/ExtractTableVerbatim.cs

@@ -10,6 +10,7 @@
 using System.Linq;
 using FAnsi.Discovery;
 using Rdmp.Core.DataExport.DataExtraction.FileOutputFormats;
+using Rdmp.Core.DataExport.DataExtraction.UserPicks;
 using Rdmp.Core.DataViewing;
 using ReusableLibraryCode.DataAccess;
 

Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExecuteFullExtractionToDatabaseMSSql.cs

@@ -471,8 +471,7 @@ protected override void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlT
         protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, DirectoryInfo lookupDir,
             IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription)
         {
-            var tbl = lookup.TableInfo.Discover(DataAccessContext.DataExport);
-            var dt = tbl.GetDataTable();
+            var dt = lookup.GetDataTable();
 
             dt.TableName = GetTableName(_destinationDatabase.Server.GetQuerySyntaxHelper().GetSensibleEntityNameFromString(lookup.TableInfo.Name));
 
@@ -490,6 +489,8 @@ protected override void TryExtractLookupTableImpl(BundledLookupTable lookup, Dir
             }
 
             destinationDb.CreateTable(dt.TableName, dt);
+
+            dt.Dispose();
         }
 
         private DiscoveredDatabase GetDestinationDatabase(IDataLoadEventListener listener)

Rdmp.Core/DataExport/DataExtraction/Pipeline/Destinations/ExtractionDestination.cs

@@ -473,9 +473,14 @@ protected virtual void TryExtractSupportingSQLTableImpl(SupportingSQLTable sqlTa
         }
 
         protected virtual void TryExtractLookupTableImpl( BundledLookupTable lookup, DirectoryInfo lookupDir, IExtractionConfiguration requestConfiguration,IDataLoadEventListener listener, out int linesWritten, out string destinationDescription)
-        {            
-            //extracts all of them
-            var extractTableVerbatim = new ExtractTableVerbatim(lookupDir, _request.Configuration.Separator, DateFormat,lookup.TableInfo.Discover(DataAccessContext.DataExport));
+        {
+            //extract the lookup table SQL
+            var sql = lookup.GetDataTableFetchSql();
+
+            var extractTableVerbatim = new ExtractTableVerbatim(
+                lookup.TableInfo.Discover(DataAccessContext.DataExport).Database.Server,
+                sql,lookup.TableInfo.GetRuntimeName(), lookupDir, _request.Configuration.Separator, DateFormat);
+
             linesWritten = extractTableVerbatim.DoExtraction();
             destinationDescription = extractTableVerbatim.OutputFilename;
         }

Rdmp.Core/DataExport/DataExtraction/UserPicks/BundledLookupTable.cs

@@ -5,7 +5,10 @@
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
+using System.Data;
 using Rdmp.Core.Curation.Data;
+using Rdmp.Core.QueryBuilding;
+using ReusableLibraryCode.DataAccess;
 
 namespace Rdmp.Core.DataExport.DataExtraction.UserPicks
 {
@@ -29,5 +32,50 @@ public override string ToString()
         {
             return TableInfo.ToString();
         }
+
+        /// <summary>
+        /// Reads lookup data from the <see cref="TableInfo"/> using <see cref="DataAccessContext.DataExport"/>
+        /// </summary>
+        /// <returns></returns>
+        public DataTable GetDataTable()
+        {
+            var tbl = TableInfo.Discover(DataAccessContext.DataExport);
+            var server = tbl.Database.Server;
+
+            var dt = new DataTable();
+
+            using (var con = server.GetConnection())
+            {
+                con.Open();
+                using (var da = server.GetDataAdapter(
+                    server.GetCommand(GetDataTableFetchSql(),con)))
+                {
+                    da.Fill(dt);
+                }
+            }
+            return dt;
+        }
+
+        public string GetDataTableFetchSql()
+        {
+            var catas = TableInfo.GetAllRelatedCatalogues();
+            QueryBuilder qb;
+
+            if (catas.Length == 1)
+            {
+                // if there is a Catalogue associated with this TableInfo use its extraction instead
+                var cata = catas[0];
+                var eis = cata.GetAllExtractionInformation(ExtractionCategory.Core);
+
+                if(eis.Length > 0)
+                {
+                    qb = new QueryBuilder(null, null, new[] { TableInfo });
+                    qb.AddColumnRange(eis);
+                    return qb.SQL;
+                }
+            }
+
+            return $"select * from {TableInfo.GetFullyQualifiedName()}";
+        }
     }
 }

Tests.Common/UnitTests.cs

@@ -598,11 +598,13 @@ private static void WhenIHaveTwoTables(MemoryDataExportRepository repository,out
         {
             ti1 = WhenIHaveA<TableInfo>(repository);
             ti1.Name = "ParentTable";
+            ti1.Database = "MyDb";
             ti1.SaveToDatabase();
             col1 = new ColumnInfo(repository, "ParentCol", "varchar(10)", ti1);
 
             ti2 = WhenIHaveA<TableInfo>(repository);
-            ti2.Name = "Child Table";
+            ti2.Name = "ChildTable";
+            ti2.Database = "MyDb";
             col2 = new ColumnInfo(repository, "ChildCol", "varchar(10)", ti2);
             col3 = new ColumnInfo(repository, "Desc", "varchar(10)", ti2);
         }