Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for PostgreSQL Transport #4443

Merged
merged 78 commits into from
Sep 27, 2024
Merged
Changes from 1 commit
Commits
Show all changes
78 commits
Select commit Hold shift + click to select a range
bd1f505
Bring sqlserver files in line with sqlserver transport to avoid confu…
jpalac Sep 12, 2024
7f28224
First changes for PostgreSql transport
jpalac Sep 12, 2024
0a13868
nullable fix
jpalac Sep 12, 2024
5ca5ca8
test fix
jpalac Sep 12, 2024
43fb83d
manifest file test fix
jpalac Sep 12, 2024
c712276
Added test project
jpalac Sep 12, 2024
a9a9fda
Do some random work on postgresql support
SzymonPobiega Sep 12, 2024
7972698
build fix
jpalac Sep 12, 2024
3feac33
fix setting value key
jpalac Sep 13, 2024
9fce98b
remove unused code
jpalac Sep 13, 2024
1376361
customise queue names with schema for postgres when provisioning queues
PhilBastian Sep 17, 2024
0c6af8b
added todo for something that errors
jpalac Sep 16, 2024
f2b5913
removed commented out code
jpalac Sep 16, 2024
c70803b
connection validator for postgres
jpalac Sep 16, 2024
e213532
Fix subscription table parameter for postgresql
jpalac Sep 16, 2024
de15819
fix test
jpalac Sep 17, 2024
3105d01
Translate transport sending queues in instances
jpalac Sep 17, 2024
5d08eda
fix test
jpalac Sep 17, 2024
0fa5744
Change ToTransportQualifiedQueueNameCore to use double quotes
jpalac Sep 17, 2024
60cc7fe
use postgres customisation for error queue names
PhilBastian Sep 17, 2024
3686c18
alternative to get latest identity value for table that doesn't rely …
PhilBastian Sep 17, 2024
2492b0f
revert double assigning of schema
PhilBastian Sep 17, 2024
c833de0
Use new alpha of the transport with fixed delay bug
andreasohlund Sep 17, 2024
17e59f4
Add connection string default
andreasohlund Sep 17, 2024
38aad49
Translate queue names when using dispatcher directly in tests
jpalac Sep 18, 2024
d6cff86
fix query to work with postgres syntax
PhilBastian Sep 18, 2024
032959d
fix syntax issues (functionality still broken)
PhilBastian Sep 18, 2024
d5e1bb6
fix RunScenario test
jpalac Sep 18, 2024
3f51022
Translate queues for forwarding message in Audit and Error ingestion
jpalac Sep 18, 2024
6a275b3
attempt to get postgres running on ci
jpalac Sep 18, 2024
cf85e92
added test category for psotgres
jpalac Sep 18, 2024
d62ee03
update postgres env variable
jpalac Sep 18, 2024
e64de3d
reverting app.config changes
jpalac Sep 18, 2024
97c851f
clarify not using nolock
PhilBastian Sep 18, 2024
e5fca39
update connection string name so it matches existing ones
jpalac Sep 18, 2024
7fe7a67
Perform az login for postgres
andreasohlund Sep 18, 2024
638cad5
fix error catching on failed connection
jpalac Sep 18, 2024
a823850
fix sqlserver tests
jpalac Sep 19, 2024
71e45f2
query fix
jpalac Sep 19, 2024
d5d0325
Fix indicator update for transports that display endpointname on SP, …
jpalac Sep 19, 2024
4d09003
Translate throughput queue for sending from monitoring
jpalac Sep 19, 2024
3a294b5
fix extra character
PhilBastian Sep 19, 2024
089236d
Add test for throughput fix
jpalac Sep 19, 2024
668f8d5
Cleanup
andreasohlund Sep 19, 2024
d19fb1d
Avoid translating audit log queue multiple times
andreasohlund Sep 19, 2024
9445e21
Avoid translating multiple times when reporting throughput
andreasohlund Sep 19, 2024
75cf270
Avoid translating error log queue multiple times
andreasohlund Sep 19, 2024
49e3c31
Remove message driven pubsub comment
andreasohlund Sep 19, 2024
0d36dcd
Fix bad merge
andreasohlund Sep 19, 2024
0709246
More bad merge fixes
andreasohlund Sep 19, 2024
5181823
Formatting
andreasohlund Sep 19, 2024
516942f
translate staging queue
jpalac Sep 20, 2024
c744a5b
fix dequeue tests
jpalac Sep 20, 2024
28c0f9c
variable name cleanup + add ability to quote subscription table
jpalac Sep 20, 2024
c17bb0d
Run container tests for postgres
andreasohlund Sep 20, 2024
a7cd4a3
Fix docker command
andreasohlund Sep 20, 2024
2485e0b
Use compose file to run postgres
andreasohlund Sep 20, 2024
21c12a4
Fix container name
andreasohlund Sep 20, 2024
db2802e
Fix pwd
andreasohlund Sep 20, 2024
48f0fd4
Fix service deps
andreasohlund Sep 20, 2024
c9c379d
Skip volume
andreasohlund Sep 20, 2024
a7b6eb1
Add health check
andreasohlund Sep 23, 2024
f6ba771
use default username and database
andreasohlund Sep 23, 2024
284efd4
Revert pool size
andreasohlund Sep 23, 2024
871e71a
Remove unneded settings
andreasohlund Sep 23, 2024
73bdd04
limit postgreSQL connections for monitoring and audit to prevent conn…
PhilBastian Sep 26, 2024
7b44827
fix rebase
jpalac Sep 26, 2024
ee98091
fix null reference in tests
jpalac Sep 26, 2024
8848a0e
more test fixing
jpalac Sep 26, 2024
fc5edf2
change name of default max concurrency level properties and apply to …
PhilBastian Sep 26, 2024
b9cf4fc
fix CI test
PhilBastian Sep 26, 2024
53b832d
revert changes to SQL server max connections defaults
PhilBastian Sep 26, 2024
ca92ae8
Decrease the queue lenght read frequency for SQL Server and PostgreSQL
SzymonPobiega Sep 26, 2024
a11bff2
Update to Postgres transport 8.1.4
andreasohlund Sep 26, 2024
2362047
Update src/ServiceControl.Transports.PostgreSql/transport.manifest
jpalac Sep 27, 2024
df4f2f6
fix from rebase
jpalac Sep 27, 2024
fdc400a
reapply change lost in rebase
PhilBastian Sep 27, 2024
036f214
reapply change lost in rebase
PhilBastian Sep 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
connection validator for postgres
jpalac committed Sep 27, 2024

Verified

This commit was signed with the committer’s verified signature.
jamesaimonetti James Aimonetti
commit c70803b5c44098edf6b0857d48ce3c522d2cf1de
3 changes: 2 additions & 1 deletion src/Directory.Packages.props
Original file line number Diff line number Diff line change
@@ -48,7 +48,8 @@
<PackageVersion Include="NServiceBus.Transport.Msmq.Sources" Version="3.0.1" />
<PackageVersion Include="NServiceBus.Transport.SqlServer" Version="8.1.3" />
<PackageVersion Include="NServiceBus.Transport.PostgreSql" Version="8.1.3" />
<PackageVersion Include="NuGet.Versioning" Version="6.11.0" />
<PackageVersion Include="Npgsql" Version="8.0.3" />
<PackageVersion Include="NuGet.Versioning" Version="6.11.0" />
<PackageVersion Include="NUnit" Version="4.2.2" />
<PackageVersion Include="NUnit.Analyzers" Version="4.3.0" />
<PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
4 changes: 2 additions & 2 deletions src/ServiceControl.Transports.PostgreSql/transport.manifest
Original file line number Diff line number Diff line change
@@ -5,9 +5,9 @@
"DisplayName": "PostgreSQL",
"AssemblyName": "ServiceControl.Transports.PostgreSql",
"TypeName": "ServiceControl.Transports.PostgreSql.PostgreSqlTransportCustomization, ServiceControl.Transports.PostgreSql",
"SampleConnectionString": "Data Source=<SQLInstance>;Initial Catalog=nservicebus;Integrated Security=True;Queue Schema=myschema;Subscriptions Table=tablename@schema@catalog",
"SampleConnectionString": "Server=<ServerName>;Database==nservicebus;Port=5432;User Id=<Username>;Password=<Password>;Queue Schema=myschema;Subscriptions Table=tablename@schema@catalog",
"AvailableInSCMU": true,
"Help": "Specify optional 'Queue Schema' to override the default schema. Specify optional 'Subscriptions Table' to override the default subscriptions table location. When integrated authentication is specified in the PostgreSql connection string the the current installing user is used to create the required SQL tables structure not the service account."
"Help": "Specify optional 'Queue Schema' to override the default schema. Specify optional 'Subscriptions Table' to override the default subscriptions table location. When integrated authentication is specified in the PostgreSql connection string the the current installing user is used to create the required SQL tables structure not the service account."
}
]
}
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
<ItemGroup>
<PackageReference Include="DotNetZip" />
<PackageReference Include="Microsoft.Data.SqlClient" />
<PackageReference Include="Npgsql" />
<PackageReference Include="NuGet.Versioning" />
<PackageReference Include="System.DirectoryServices.AccountManagement" />
<PackageReference Include="System.Management" />
Original file line number Diff line number Diff line change
@@ -5,7 +5,8 @@
using System.Linq;
using Accounts;
using Microsoft.Data.SqlClient;
//TODO postgres
using Npgsql;

class ConnectionStringValidator
{
ConnectionStringValidator(string connectionString, string serviceAccount)
@@ -21,6 +22,10 @@ public static void Validate(IServiceControlAuditInstance instance)
{
validator.CheckMsSqlConnectionString();
}
else if (instance.TransportPackage.Name == "PostgreSQL")
{
validator.CheckPostgreSqlConnectString();
}
}

public static void Validate(IServiceControlInstance instance)
@@ -30,6 +35,10 @@ public static void Validate(IServiceControlInstance instance)
{
validator.CheckMsSqlConnectionString();
}
else if (instance.TransportPackage.Name == "PostgreSQL")
{
validator.CheckPostgreSqlConnectString();
}
}

public static void Validate(IMonitoringInstance instance)
@@ -39,6 +48,10 @@ public static void Validate(IMonitoringInstance instance)
{
validator.CheckMsSqlConnectionString();
}
else if (instance.TransportPackage.Name == "PostgreSQL")
{
validator.CheckPostgreSqlConnectString();
}
}

void CheckMsSqlConnectionString()
@@ -96,6 +109,42 @@ void CheckMsSqlConnectionString()
}
}

//TODO postgres - do we actually look at the 'Subscriptions Table' custom key anywhere?
void CheckPostgreSqlConnectString()
{
string[] customKeys = { "Queue Schema", "Subscriptions Table" };

try
{
//Check validity of connection string. This will throw if invalid
var builder = new DbConnectionStringBuilder { ConnectionString = connectionString };

//The NSB PostgreSQL Transport can have custom key/value pairs in the connection string
// that won't make sense to PostgreSQL. Remove these from the string we want to validate.
foreach (var customKey in customKeys)
{
if (builder.ContainsKey(customKey))
{
builder.Remove(customKey);
}
}

//Attempt to connect to DB
using (var s = new NpgsqlConnection(builder.ConnectionString))
{
s.Open();
}
}
catch (ArgumentException argumentException)
{
throw new EngineValidationException($"Connection String is invalid - {argumentException.Message}");
}
catch (SqlException sqlEx)
{
throw new EngineValidationException($"PostgreSQL connection failed - {sqlEx.Message}");
}
}

string connectionString;
string serviceAccount;
}