Skip to content

Commit

Permalink
Upgrade dependencies to modern SQLServer and PGSQL drivers
Browse files Browse the repository at this point in the history
Added NEtDateTimeKind to address TZ/noTZ ambiguity
Address https://www.npgsql.org/doc/release-notes/6.0.html#major-changes-to-timestamp-mapping incopatibility
Address dotnet/SqlClient#1402 new SQL Server encryption defaults
  • Loading branch information
rpsft committed Jan 3, 2022
1 parent eb578fd commit 3bdd03b
Show file tree
Hide file tree
Showing 29 changed files with 439 additions and 332 deletions.
20 changes: 10 additions & 10 deletions ETLBox/ETLBox.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,19 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="CsvHelper" Version="27.1.1" />
<PackageReference Include="CsvHelper" Version="27.2.1" />
<PackageReference Include="ExcelDataReader" Version="3.6.0" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="1.1.2" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="4.0.0" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.1" />
<PackageReference Include="MySql.Data" Version="8.0.19" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="NLog" Version="4.7.0" />
<PackageReference Include="NLog.Extensions.Logging" Version="1.6.2" />
<PackageReference Include="Npgsql" Version="4.1.3.1" />
<PackageReference Include="MySql.Data" Version="8.0.27" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
<PackageReference Include="NLog" Version="4.7.13" />
<PackageReference Include="NLog.Extensions.Logging" Version="1.7.4" />
<PackageReference Include="Npgsql" Version="6.0.2" />
<PackageReference Include="System.Data.Common" Version="4.3.0" />
<PackageReference Include="System.Data.Odbc" Version="4.7.0" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.0" />
<PackageReference Include="TSQL.Parser" Version="1.5.0" />
<PackageReference Include="System.Data.Odbc" Version="6.0.0" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0.0" />
<PackageReference Include="TSQL.Parser" Version="1.5.3" />
<PackageReference Include="Unofficial.Microsoft.AnalysisServices.AdomdClientNetCore" Version="15.3.1.2" />
</ItemGroup>

Expand Down
19 changes: 19 additions & 0 deletions ETLBox/src/Definitions/ConnectionManager/DataTypeConverter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -165,5 +165,24 @@ public static string TryGetDBSpecificType(string dbSpecificTypeName, ConnectionM
return dbSpecificTypeName;
}
}

public static DateTimeKind? GetNETDateTimeKind(string dbSpecificTypeName)
{
switch (dbSpecificTypeName)
{
case "date":
case "datetime":
case "smalldatetime":
case "datetime2":
case "time":
case "timestamp":
return DateTimeKind.Unspecified;
case "timetz":
case "timestamptz":
return DateTimeKind.Utc;
default:
return null;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,10 @@ public IDbCommand CreateCommand(string commandText, IEnumerable<QueryParameter>
{
var newPar = cmd.CreateParameter();
newPar.ParameterName = par.Name;
newPar.DbType = par.DBType;
newPar.DbType = ConnectionManagerType == ConnectionManagerType.Postgres && par.DBType == DbType.DateTime
? DbType.DateTime2 // TODO: Move this hack to more appropriate place
// (fix for https://www.npgsql.org/doc/release-notes/6.0.html#major-changes-to-timestamp-mapping in NpgSql 6.0+)
: par.DBType;
newPar.Value = par.Value;
cmd.Parameters.Add(newPar);
}
Expand Down
3 changes: 2 additions & 1 deletion ETLBox/src/Definitions/Database/TableColumn.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@ public class TableColumn : ITableColumn, IColumnMapping
public string DefaultValue { get; set; }
public string Collation { get; set; }
public string ComputedColumn { get; set; }
public bool HasComputedColumn => !String.IsNullOrWhiteSpace(ComputedColumn);
public bool HasComputedColumn => !string.IsNullOrWhiteSpace(ComputedColumn);
public System.Type NETDataType => Type.GetType(DataTypeConverter.GetNETObjectTypeString(DataType));
public DateTimeKind? NETDateTimeKind => DataTypeConverter.GetNETDateTimeKind(DataType);

public string Comment { get; set; } //MySql only
public int? IdentitySeed { get; set; } //Sql Server only
Expand Down
15 changes: 11 additions & 4 deletions TestConnectionManager/TestConnectionManager.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,17 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="xunit" Version="2.4.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
<PackageReference Include="JunitXml.TestLogger" Version="3.0.110" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="3.1.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
Expand Down
8 changes: 4 additions & 4 deletions TestConnectionManager/default.config.json-template
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"DataFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};TrustServerCertificate=true;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlow",
"SqlOdbcConnectionString": "Driver={SQL Server};Server=${CFG_MSSQL_IP};Database=ETLBox_DataFlow;Uid=sa;Pwd=YourStrong@Passw0rd;",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessDataFlow.accdb",
Expand All @@ -9,18 +9,18 @@
"AzureSqlConnectionString": "Server=tcp:etlbox.database.windows.net,1433;Initial Catalog=etlbox;Persist Security Info=False;User ID=etlboxadmin;Password=ETLBoxPassw0rd!;"
},
"DataFlowSource": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource",
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};TrustServerCertificate=true;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowSource",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowSource.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowSource;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowSource;User Id=postgres;Password=etlboxpassword;"
},
"DataFlowDestination": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination",
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};TrustServerCertificate=true;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_DataFlowDestination",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteDataFlowDestination.db;",
"MySqlConnectionString": "Server=${CFG_MYSQL_IP};Database=ETLBox_DataFlowDestination;Uid=root;Pwd=etlboxpassword;",
"PostgresConnectionString": "Server=${CFG_POSTGRES_IP};Database=ETLBox_DataFlowDestination;User Id=postgres;Password=etlboxpassword;"
},
"ConnectionManager": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ConnectionManager"
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};TrustServerCertificate=true;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ConnectionManager"
}
}
15 changes: 11 additions & 4 deletions TestControlFlowTasks/TestControlFlowTasks.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,17 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="xunit" Version="2.4.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
<PackageReference Include="JunitXml.TestLogger" Version="3.0.110" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="3.1.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion TestControlFlowTasks/default.config.json-template
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"ControlFlow": {
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ControlFlow",
"SqlConnectionString": "Data Source=${CFG_MSSQL_IP};TrustServerCertificate=true;User Id=sa;Password=YourStrong@Passw0rd;Initial Catalog=ETLBox_ControlFlow",
"SSASConnectionString": "Data Source=.;Integrated Security=SSPI;Initial Catalog=ETLBox_ControlFlow",
"SQLiteConnectionString": "Data Source=.${CFG_PD}db${CFG_PD}SQLiteControlFlow.db;",
"AccessOdbcConnectionString": "Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=.${CFG_PD}db${CFG_PD}AccessControlFlow.accdb",
Expand Down
10 changes: 5 additions & 5 deletions TestControlFlowTasks/src/Postgres/TableDefinitionTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ tswithtimezone TIMESTAMPTZ
//Assert
Assert.Collection(result.Columns,
tc => Assert.True(tc.DataType == "date" && tc.NETDataType == typeof(DateTime)),
tc => Assert.True(tc.DataType == "time" && tc.NETDataType == typeof(DateTime)),
tc => Assert.True(tc.DataType == "timetz" && tc.NETDataType == typeof(DateTime)),
tc => Assert.True(tc.DataType == "interval" && tc.NETDataType == typeof(String)),
tc => Assert.True(tc.DataType == "timestamp" && tc.NETDataType == typeof(DateTime)),
tc => Assert.True(tc.DataType == "timestamptz" && tc.NETDataType == typeof(DateTime))
tc => Assert.True(tc.DataType == "time" && tc.NETDataType == typeof(DateTime) && tc.NETDateTimeKind == DateTimeKind.Unspecified),
tc => Assert.True(tc.DataType == "timetz" && tc.NETDataType == typeof(DateTime) && tc.NETDateTimeKind == DateTimeKind.Utc),
tc => Assert.True(tc.DataType == "interval" && tc.NETDataType == typeof(String) && tc.NETDateTimeKind == null),
tc => Assert.True(tc.DataType == "timestamp" && tc.NETDataType == typeof(DateTime) && tc.NETDateTimeKind == DateTimeKind.Unspecified),
tc => Assert.True(tc.DataType == "timestamptz" && tc.NETDataType == typeof(DateTime) && tc.NETDateTimeKind == DateTimeKind.Utc)
);
}

Expand Down
117 changes: 61 additions & 56 deletions TestControlFlowTasks/src/SqlTaskBulkInsertTests.cs
Original file line number Diff line number Diff line change
@@ -1,82 +1,87 @@
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using ALE.ETLBox;
using ALE.ETLBox.ConnectionManager;
using ALE.ETLBox.ControlFlow;
using ALE.ETLBox.Helper;
using ALE.ETLBox.Logging;
using ALE.ETLBoxTests.Fixtures;
using System;
using System.Collections.Generic;
using Xunit;

namespace ALE.ETLBoxTests.ControlFlowTests
namespace ALE.ETLBoxTests.ControlFlowTests;

[Collection("ControlFlow")]
public class SqlTaskBulkInsertTests
{
[Collection("ControlFlow")]
public class SqlTaskBulkInsertTests
public SqlTaskBulkInsertTests(ControlFlowDatabaseFixture dbFixture)
{
}

public static IEnumerable<object[]> Connections => RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? Config.AllSqlConnections("ControlFlow").Concat(Config.AccessConnection("ControlFlow"))
: Config.AllSqlConnections("ControlFlow");

public static IEnumerable<object[]> ConnectionsWithValue(int value)
{
return Config.AllSqlConnectionsWithValue("ControlFlow", value);
}


[Theory]
[MemberData(nameof(Connections))]
//If access fails with "Internal OLE Automation error", download and install: https://www.microsoft.com/en-us/download/confirmation.aspx?id=50040
//see also: https://stackoverflow.com/questions/54632928/internal-ole-automation-error-in-ms-access-using-oledb
public void StringArray(IConnectionManager connection)
{
public static IEnumerable<object[]> Connections => Config.AllSqlConnections("ControlFlow");
public static IEnumerable<object[]> ConnectionsWithValue(int value)
=> Config.AllSqlConnectionsWithValue("ControlFlow", value);
public static IEnumerable<object[]> Access => Config.AccessConnection("ControlFlow");
//Arrange
var destTable = new TwoColumnsTableFixture(connection, "BulkInsert2Columns");

public SqlTaskBulkInsertTests(ControlFlowDatabaseFixture dbFixture)
{ }
var data = new TableData<string[]>(destTable.TableDefinition);
string[] values = { "1", "Test1" };
data.Rows.Add(values);
string[] values2 = { "2", "Test2" };
data.Rows.Add(values2);
string[] values3 = { "3", "Test3" };
data.Rows.Add(values3);

//Act
SqlTask.BulkInsert(connection, "Bulk insert demo data", data, "BulkInsert2Columns");

[Theory, MemberData(nameof(Connections))
, MemberData(nameof(Access))] //If access fails with "Internal OLE Automation error", download and install: https://www.microsoft.com/en-us/download/confirmation.aspx?id=50040
//see also: https://stackoverflow.com/questions/54632928/internal-ole-automation-error-in-ms-access-using-oledb
//Assert
destTable.AssertTestData();

public void StringArray(IConnectionManager connection)
if (connection.GetType() == typeof(AccessOdbcConnectionManager))
connection.Close();
//Assert connection is closed
Assert.True(connection.State == null);
}

[Theory]
[MemberData(nameof(ConnectionsWithValue), 0)]
[MemberData(nameof(ConnectionsWithValue), 2)]
[MemberData(nameof(ConnectionsWithValue), 3)]
public void WithIdentityShift(IConnectionManager connection, int identityIndex)
{
//SQLite does not support Batch Insert on Non Nullable Identity Columns
if (connection.GetType() != typeof(SQLiteConnectionManager))
{
//Arrange
TwoColumnsTableFixture destTable = new TwoColumnsTableFixture(connection, "BulkInsert2Columns");
var destTable =
new FourColumnsTableFixture(connection, "BulkInsert4Columns", identityIndex);

TableData<string[]> data = new TableData<string[]>(destTable.TableDefinition);
string[] values = { "1", "Test1" };
var data = new TableData(destTable.TableDefinition, 2);
object[] values = { "Test1", null, 1.2 };
data.Rows.Add(values);
string[] values2 = { "2", "Test2" };
object[] values2 = { "Test2", 4711, 1.23 };
data.Rows.Add(values2);
string[] values3 = { "3", "Test3" };
object[] values3 = { "Test3", 185, 1.234 };
data.Rows.Add(values3);

//Act
SqlTask.BulkInsert(connection, "Bulk insert demo data", data, "BulkInsert2Columns");
SqlTask.BulkInsert(connection, "Bulk insert demo data", data, "BulkInsert4Columns");

//Assert
destTable.AssertTestData();

if (connection.GetType() == typeof(AccessOdbcConnectionManager))
connection.Close();
//Assert connection is closed
Assert.True(connection.State == null);
}

[Theory, MemberData(nameof(ConnectionsWithValue), 0),
MemberData(nameof(ConnectionsWithValue), 2),
MemberData(nameof(ConnectionsWithValue), 3)]
public void WithIdentityShift(IConnectionManager connection, int identityIndex)
{
//SQLite does not support Batch Insert on Non Nullable Identity Columns
if (connection.GetType() != typeof(SQLiteConnectionManager))
{
//Arrange
FourColumnsTableFixture destTable = new FourColumnsTableFixture(connection, "BulkInsert4Columns", identityIndex);

TableData data = new TableData(destTable.TableDefinition, 2);
object[] values = { "Test1", null, 1.2 };
data.Rows.Add(values);
object[] values2 = { "Test2", 4711, 1.23 };
data.Rows.Add(values2);
object[] values3 = { "Test3", 185, 1.234 };
data.Rows.Add(values3);

//Act
SqlTask.BulkInsert(connection, "Bulk insert demo data", data, "BulkInsert4Columns");

//Assert
destTable.AssertTestData();
}
}

}
}
}
Loading

0 comments on commit 3bdd03b

Please sign in to comment.