diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index d3fb7abf7e02..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-mssql-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" - config_path: "src/test/resources/config.json" diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle deleted file mode 100644 index 3fccfd3edea4..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'application' - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.6.2' - features = ['db-sources'] - useLocalCdk = false -} - -configurations.all { - resolutionStrategy { - force libs.jooq - } -} - -application { - mainClass = 'io.airbyte.integrations.source.mssql_strict_encrypt.MssqlSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-integrations:connectors:source-mssql') - implementation libs.jooq - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-mssql')) - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.testcontainers.mssqlserver - testImplementation 'org.hamcrest:hamcrest-all:1.3' -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/source-mssql-strict-encrypt/gradle.properties deleted file mode 100644 index 8ef098d20b92..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -testExecutionConcurrency=-1 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg deleted file mode 100644 index edcaeb77c8f2..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml deleted file mode 100644 index 90684c5f2949..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,32 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 3.0.2 - dockerRepository: airbyte/source-mssql-strict-encrypt - githubIssueLabel: source-mssql - icon: mssql.svg - license: ELv2 - name: Microsoft SQL Server (MSSQL) - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mssql - tags: - - language:java - releases: - breakingChanges: - 3.0.0: - message: "Remapped columns of types: date, datetime, datetime2, datetimeoffset, smalldatetime, and time from `String` to their appropriate Airbyte types. Customers whose streams have columns with the affected data types must take action with their connections." - upgradeDeadline: "2023-12-07" - 2.0.0: - message: "Add default cursor for cdc" - upgradeDeadline: "2023-08-23" -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlSourceStrictEncrypt.java deleted file mode 100644 index 8687b6c81822..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlSourceStrictEncrypt.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql_strict_encrypt; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.base.Source; -import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.mssql.MssqlSource; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MssqlSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(MssqlSourceStrictEncrypt.class); - - public MssqlSourceStrictEncrypt() { - super(MssqlSource.sshWrappedSource(new MssqlSource())); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) throws Exception { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - ((ArrayNode) spec.getConnectionSpecification().get("properties").get("ssl_method").get("oneOf")).remove(0); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new MssqlSourceStrictEncrypt(); - LOGGER.info("starting source: {}", MssqlSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MssqlSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index c584e76113cd..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql_strict_encrypt; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.base.ssh.SshHelpers; -import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.source.mssql.MsSQLContainerFactory; -import io.airbyte.integrations.source.mssql.MsSQLTestDatabase; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.HashMap; -import java.util.Map; - -public class MssqlStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { - - protected static final String SCHEMA_NAME = "dbo"; - protected static final String STREAM_NAME = "id_and_name"; - - private MsSQLTestDatabase testdb; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) { - final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-RTM-CU2-ubuntu-20.04"); - testdb = new MsSQLTestDatabase(container); - testdb = testdb - .withConnectionProperty("encrypt", "true") - .withConnectionProperty("trustServerCertificate", "true") - .withConnectionProperty("databaseName", testdb.getDatabaseName()) - .initialized() - .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") - .with("INSERT INTO id_and_name (id, name, born) VALUES " + - "(1,'picard', '2124-03-04T01:01:01Z'), " + - "(2, 'crusher', '2124-03-04T01:01:01Z'), " + - "(3, 'vash', '2124-03-04T01:01:01Z');"); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - testdb.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-mssql-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - } - - @Override - protected JsonNode getConfig() { - return testdb.integrationTestConfigBuilder() - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) - .build(); - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return CatalogHelpers.createConfiguredAirbyteCatalog( - STREAM_NAME, - SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING), - Field.of("born", JsonSchemaType.STRING)); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptJdbcSourceAcceptanceTest.java deleted file mode 100644 index 2aac6a760c84..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql_strict_encrypt/MssqlStrictEncryptJdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql_strict_encrypt; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.integrations.base.ssh.SshHelpers; -import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.source.mssql.MsSQLContainerFactory; -import io.airbyte.integrations.source.mssql.MsSQLTestDatabase; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; - -public class MssqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - static { - // In mssql, timestamp is generated automatically, so we need to use - // the datetime type instead so that we can set the value manually. - COL_TIMESTAMP_TYPE = "DATETIME"; - } - - @Override - protected void maybeSetShorterConnectionTimeout(final JsonNode config) { - ((ObjectNode) config).put(JdbcUtils.JDBC_URL_PARAMS_KEY, "loginTimeout=1"); - } - - protected JsonNode config() { - return testdb.testConfigBuilder() - .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) - .build(); - } - - @Override - protected MssqlSourceStrictEncrypt source() { - return new MssqlSourceStrictEncrypt(); - } - - @Override - protected MsSQLTestDatabase createTestDatabase() { - final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-RTM-CU2-ubuntu-20.04"); - final var testdb = new MsSQLTestDatabase(container); - return testdb - .withConnectionProperty("encrypt", "true") - .withConnectionProperty("trustServerCertificate", "true") - .withConnectionProperty("databaseName", testdb.getDatabaseName()) - .initialized(); - } - - @Override - public boolean supportsSchemas() { - return true; - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = source().spec(); - final ConnectorSpecification expected = - SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - - assertEquals(expected, actual); - } - - @Override - protected AirbyteCatalog getCatalog(final String defaultNamespace) { - return new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(Collections.emptyList()), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey( - List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/config.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/config.json deleted file mode 100644 index c8ad0c9c0735..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "host": "default", - "port": 5555, - "database": "default", - "username": "default", - "ssl_method": { "ssl_method": "encrypted_trust_server_certificate" }, - "replication_method": { "method": "STANDARD" } -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json deleted file mode 100644 index 607be5d3fae1..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ /dev/null @@ -1,274 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MSSQL Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The hostname of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "examples": ["1433"], - "order": 1 - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["master"], - "order": 2 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["dbo"], - "order": 3 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 4 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 6 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used when communicating with the database.", - "order": 7, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Update Method", - "description": "Configures how data is extracted from the database.", - "default": "CDC", - "display_type": "radio", - "order": 8, - "oneOf": [ - { - "title": "Read Changes using Change Data Capture (CDC)", - "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the SQL Server's change data capture feature. This must be enabled on your database.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 0 - }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "min": 120, - "max": 1200, - "order": 3 - } - } - }, - { - "title": "Scan Changes with User Defined Cursor", - "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index e84ae6faba11..80d6fe17c242 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 3.0.2 + dockerImageTag: 3.1.0 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 5ce64b942485..d5d66250e2fb 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -27,6 +27,7 @@ import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; @@ -41,6 +42,7 @@ import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -86,6 +88,10 @@ SELECT CAST(IIF(EXISTS(SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL), 1, 0) private static final String HIERARCHYID = "hierarchyid"; private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; + public static final String TUNNEL_METHOD = "tunnel_method"; + public static final String NO_TUNNEL = "NO_TUNNEL"; + public static final String SSL_METHOD = "ssl_method"; + public static final String SSL_METHOD_UNENCRYPTED = "unencrypted"; private List schemas; public static Source sshWrappedSource(MssqlSource source) { @@ -96,6 +102,29 @@ public MssqlSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new MssqlSourceOperations()); } + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly + // and not over SSH tunnel + if (cloudDeploymentMode()) { + if (config.has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { + // If no SSH tunnel. + if (config.has(SSL_METHOD) && config.get(SSL_METHOD).has(SSL_METHOD) && + SSL_METHOD_UNENCRYPTED.equalsIgnoreCase(config.get(SSL_METHOD).get(SSL_METHOD).asText())) { + // Fail in case SSL method is unencrypted. + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Unsecured connection not allowed. " + + "If no SSH Tunnel set up, please use one of the following SSL methods: " + + "encrypted_trust_server_certificate, encrypted_verify_certificate."); + } + } + } + return super.check(config); + } + @Override public AutoCloseableIterator queryTableFullRefresh(final JdbcDatabase database, final List columnNames, @@ -569,6 +598,10 @@ private void readSsl(final JsonNode sslMethod, final List additionalPara } } + private boolean cloudDeploymentMode() { + return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); + } + public static void main(final String[] args) throws Exception { final Source source = MssqlSource.sshWrappedSource(new MssqlSource()); LOGGER.info("starting source: {}", MssqlSource.class); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..3be550d9a8d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import java.util.Map; + +public class CloudDeploymentSslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTest { + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) { + final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-RTM-CU2-ubuntu-20.04"); + testdb = new MsSQLTestDatabase(container); + testdb = testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("trustServerCertificate", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized() + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1,'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');"); + } + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags(), "CLOUD"); + } + + @Override + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json deleted file mode 100644 index a00b685e6f06..000000000000 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MSSQL Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The hostname of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "examples": ["1433"], - "order": 1 - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["master"], - "order": 2 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["dbo"], - "order": 3 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 4 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 6 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used when communicating with the database.", - "order": 7, - "oneOf": [ - { - "title": "Unencrypted", - "description": "Data transfer will not be encrypted.", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "unencrypted" - } - } - }, - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Update Method", - "description": "Configures how data is extracted from the database.", - "default": "CDC", - "display_type": "radio", - "order": 8, - "oneOf": [ - { - "title": "Read Changes using Change Data Capture (CDC)", - "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the SQL Server's change data capture feature. This must be enabled on your database.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 0 - }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "min": 120, - "max": 1200, - "order": 3 - } - } - }, - { - "title": "Scan Changes with User Defined Cursor", - "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java new file mode 100644 index 000000000000..fcc8ea0ff2f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +@Execution(ExecutionMode.CONCURRENT) +public class CloudDeploymentMssqlTest { + + private MsSQLTestDatabase createTestDatabase(String... containerFactoryMethods) { + final var container = new MsSQLContainerFactory().shared( + "mcr.microsoft.com/mssql/server:2022-RTM-CU2-ubuntu-20.04", containerFactoryMethods); + final var testdb = new MsSQLTestDatabase(container); + return testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("trustServerCertificate", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized(); + } + + private Source source() { + final var source = new MssqlSource(); + source.setFeatureFlags(FeatureFlagsWrapper.overridingDeploymentMode(new EnvVariableFeatureFlags(), "CLOUD")); + return MssqlSource.sshWrappedSource(source); + } + + @Test + void testStrictSSLUnsecuredNoTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withoutSsl() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Unsecured connection not allowed"), actual.getMessage()); + } + } + + @Test + void testStrictSSLSecuredNoTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.testConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } + } + + @Test + void testStrictSSLSecuredWithTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testStrictSSLUnsecuredWithTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")) + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testCheckWithSslModeDisabled() throws Exception { + try (final var testdb = createTestDatabase("withNetwork")) { + try (final SshBastionContainer bastion = new SshBastionContainer()) { + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); + final var config = testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, false)) + .withoutSsl() + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } + } + } + +} diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 0b353394de2e..b0873995c68e 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -342,6 +342,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.1.0 | 2023-11-28 | [32882](https://github.com/airbytehq/airbyte/pull/32882) | enforce SSL on Airbyte Cloud | | 3.0.2 | 2023-11-27 | [32573](https://github.com/airbytehq/airbyte/pull/32573) | Format Datetime and Datetime2 datatypes to 6-digit microsecond precision | | 3.0.1 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | | 3.0.0 | 2023-11-07 | [31531](https://github.com/airbytehq/airbyte/pull/31531) | Remapped date, smalldatetime, datetime2, time, and datetimeoffset datatype to their correct Airbyte types |