CockroachDb source: Add only encrypted version for the connector (air…

…bytehq#7457)

* [6717] Add only encrypted version for the CockroachDb

* format

* Create custom docker for CockroachDB with SSL

* add custom docker + test

* enable container

* set Cockroach version and add container to integration test

* fix docker

* remove test

* fix version

* fix container instancing scope

* left only integration test

* remove ssl from expected spec

* fix urls

* clean import

* chang log + build

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -954,7 +954,7 @@
     supported_destination_sync_modes: []
 - dockerImage: "airbyte/source-cockroachdb:0.1.3"
   spec:
-    documentationUrl: "https://docs.airbyte.io/integrations/sources/postgres"
+    documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Cockroach Source Spec"

airbyte-db/jooq/build.gradle

@@ -6,7 +6,7 @@ plugins {
 dependencies {
     implementation 'org.jooq:jooq-meta:3.13.4'
     implementation 'org.jooq:jooq:3.13.4'
-    implementation 'org.postgresql:postgresql:42.2.18'
+    implementation 'org.postgresql:postgresql:42.3.1'
     implementation "org.flywaydb:flyway-core:7.14.0"
 
     implementation project(':airbyte-db:lib')

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile

@@ -0,0 +1,12 @@
+FROM airbyte/integration-base-java:dev
+
+WORKDIR /airbyte
+
+ENV APPLICATION source-cockroachdb-strict-encrypt
+
+COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
+
+RUN tar xf ${APPLICATION}.tar --strip-components=1
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/acceptance-test-config.yml

@@ -0,0 +1,6 @@
+# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
+# for more information about how to configure these tests
+connector_image: airbyte/source-cockroachdb-strict-encrypt:dev
+tests:
+  spec:
+    - spec_path: "src/test/resources/expected_spec.json"

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle

@@ -0,0 +1,31 @@
+plugins {
+    id 'application'
+    id 'airbyte-docker'
+    id 'airbyte-integration-test-java'
+}
+
+application {
+    mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSourceStrictEncrypt'
+    applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0']
+}
+
+dependencies {
+    implementation project(':airbyte-db:lib')
+    implementation project(':airbyte-integrations:bases:base-java')
+    implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-integrations:connectors:source-jdbc')
+    implementation project(':airbyte-integrations:connectors:source-relational-db')
+    implementation project(':airbyte-integrations:connectors:source-cockroachdb')
+
+    implementation "org.testcontainers:testcontainers:1.15.3"
+    implementation "org.testcontainers:jdbc:1.15.3"
+    implementation "org.testcontainers:cockroachdb:1.15.3"
+    implementation "org.postgresql:postgresql:42.3.1"
+
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb')
+    integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb-strict-encrypt')
+    integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test')
+
+    implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+    integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs)
+}

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceStrictEncrypt.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.source.cockroachdb;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.integrations.base.IntegrationRunner;
+import io.airbyte.integrations.base.Source;
+import io.airbyte.integrations.base.spec_modification.SpecModifyingSource;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CockroachDbSourceStrictEncrypt extends SpecModifyingSource implements Source {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(CockroachDbSourceStrictEncrypt.class);
+
+  public CockroachDbSourceStrictEncrypt() {
+    super(CockroachDbSource.sshWrappedSource());
+  }
+
+  @Override
+  public ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception {
+    final ConnectorSpecification spec = Jsons.clone(originalSpec);
+    ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove("ssl");
+    return spec;
+  }
+
+  public static void main(final String[] args) throws Exception {
+    final Source source = new CockroachDbSourceStrictEncrypt();
+    LOGGER.info("starting source: {}", CockroachDbSourceStrictEncrypt.class);
+    new IntegrationRunner(source).run(args);
+    LOGGER.info("completed source: {}", CockroachDbSourceStrictEncrypt.class);
+  }
+
+}

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSslTestContainer.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.source.cockroachdb;
+
+import java.util.concurrent.TimeUnit;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+
+public class CockroachDbSslTestContainer {
+
+  private GenericContainer cockroachSslDbContainer;
+
+  public void start() throws Exception {
+    if (cockroachSslDbContainer != null)
+      cockroachSslDbContainer.stop();
+
+    Network network = Network.newNetwork();
+    cockroachSslDbContainer = new GenericContainer(
+        new ImageFromDockerfile("cockroach-test")
+            .withFileFromClasspath("Dockerfile", "docker/Dockerfile")
+            .withFileFromClasspath("cockroachdb_init.sh", "docker/cockroachdb_init.sh")
+            .withFileFromClasspath("cockroachdb_test_user.sh", "docker/cockroachdb_test_user.sh"))
+                .withNetwork(network)
+                .withExposedPorts(26257);
+    cockroachSslDbContainer.start();
+
+    // Wait till test user is created
+    TimeUnit.SECONDS.sleep(5);
+  }
+
+  public void close() {
+    cockroachSslDbContainer.stop();
+    cockroachSslDbContainer = null;
+  }
+
+  public GenericContainer getCockroachSslDbContainer() {
+    return cockroachSslDbContainer;
+  }
+
+}

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/Dockerfile

@@ -0,0 +1,22 @@
+FROM cockroachdb/cockroach:v20.2.18
+
+##
+# Test CockroachDB container with enabled SSL
+# Database: defaultdb
+# User: test_user
+# Build command: docker build -f Dockerfile -t cockroachdb-test-ssl:latest .
+# Run command: docker run -td -p 26257:26257 -p 8080:8080 --name cockroach-test-cont  cockroachdb-test-ssl:latest
+##
+
+ENV COCKROACH_HOST localhost:26257
+ENV COCKROACH_CERTS_DIR certs
+
+EXPOSE 8080
+EXPOSE 26257
+
+COPY cockroachdb_init.sh .
+COPY cockroachdb_test_user.sh .
+
+RUN chmod +x cockroachdb_init.sh
+
+ENTRYPOINT cockroachdb_init.sh

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_init.sh

@@ -0,0 +1,12 @@
+echo "Preparing certs"
+mkdir "certs"
+mkdir "safe-place"
+cockroach cert create-ca --certs-dir=certs --ca-key=safe-place/ca.key
+cockroach cert create-node localhost "$HOSTNAME" --certs-dir=certs --ca-key=safe-place/ca.key
+cockroach cert create-client root --certs-dir=certs --ca-key=safe-place/ca.key
+cockroach cert create-client test_user --certs-dir=certs --ca-key=safe-place/ca.key
+echo "Finished preparing certs"
+
+echo "Starting CockroachDB"
+nohup sh cockroachdb_test_user.sh &
+cockroach start-single-node --certs-dir=certs

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_test_user.sh

@@ -0,0 +1,13 @@
+countAttempt=0
+maxAttempt=10
+while : ; do
+  cockroach sql -e "create user test_user with password 'test_user'" &> user_creation.log && cockroach sql -e "GRANT ALL ON DATABASE defaultdb TO test_user" &>> user_creation.log
+  ((countAttempt++))
+
+  if [ "$countAttempt" = "$maxAttempt" ] || [ "$(grep -c 'ERROR' user_creation.log)" = 0 ]; then
+    echo "User test_user is created with grants to defaultdb! Login=Pass"
+    break
+  fi
+  sleep 2
+  echo "Attempt #$countAttempt to create a user is failed!"
+done

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json

@@ -0,0 +1,47 @@
+{
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/cockroachdb",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Cockroach Source Spec",
+    "type": "object",
+    "required": ["host", "port", "database", "username"],
+    "additionalProperties": false,
+    "properties": {
+      "host": {
+        "title": "Host",
+        "description": "Hostname of the database.",
+        "type": "string",
+        "order": 0
+      },
+      "port": {
+        "title": "Port",
+        "description": "Port of the database.",
+        "type": "integer",
+        "minimum": 0,
+        "maximum": 65536,
+        "default": 5432,
+        "examples": ["5432"],
+        "order": 1
+      },
+      "database": {
+        "title": "DB Name",
+        "description": "Name of the database.",
+        "type": "string",
+        "order": 2
+      },
+      "username": {
+        "title": "User",
+        "description": "Username to use to access the database.",
+        "type": "string",
+        "order": 3
+      },
+      "password": {
+        "title": "Password",
+        "description": "Password associated with the username.",
+        "type": "string",
+        "airbyte_secret": true,
+        "order": 4
+      }
+    }
+  }
+}

airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.integrations.source.cockroachdb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import io.airbyte.commons.json.Jsons;
+import io.airbyte.commons.resources.MoreResources;
+import io.airbyte.db.Database;
+import io.airbyte.db.Databases;
+import io.airbyte.integrations.base.ssh.SshHelpers;
+import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
+import io.airbyte.integrations.standardtest.source.TestDestinationEnv;
+import io.airbyte.protocol.models.CatalogHelpers;
+import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
+import io.airbyte.protocol.models.ConfiguredAirbyteStream;
+import io.airbyte.protocol.models.ConnectorSpecification;
+import io.airbyte.protocol.models.DestinationSyncMode;
+import io.airbyte.protocol.models.Field;
+import io.airbyte.protocol.models.JsonSchemaPrimitive;
+import io.airbyte.protocol.models.SyncMode;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import org.jooq.SQLDialect;
+
+public class CockroachDbEncryptSourceAcceptanceTest extends SourceAcceptanceTest {
+
+  private static final String STREAM_NAME = "public.id_and_name";
+  private static final String STREAM_NAME2 = "public.starships";
+
+  private CockroachDbSslTestContainer container;
+  private JsonNode config;
+
+  @Override
+  protected void setupEnvironment(final TestDestinationEnv environment) throws Exception {
+    container = new CockroachDbSslTestContainer();
+    container.start();
+
+    config = Jsons.jsonNode(ImmutableMap.builder()
+        .put("host", container.getCockroachSslDbContainer().getHost())
+        .put("port", container.getCockroachSslDbContainer().getFirstMappedPort())
+        .put("database", "defaultdb")
+        .put("username", "test_user")
+        .put("password", "test_user")
+        .build());
+
+    final Database database = Databases.createDatabase(
+        config.get("username").asText(),
+        config.get("password").asText(),
+        String.format("jdbc:postgresql://%s:%s/%s",
+            config.get("host").asText(),
+            config.get("port").asText(),
+            config.get("database").asText()),
+        "org.postgresql.Driver",
+        SQLDialect.POSTGRES);
+
+    database.query(ctx -> {
+      ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));");
+      ctx.fetch(
+          "INSERT INTO id_and_name (id, name) VALUES (1,'picard'),  (2, 'crusher'), (3, 'vash');");
+      ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));");
+      ctx.fetch(
+          "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'),  (2, 'defiant'), (3, 'yamato');");
+      return null;
+    });
+
+    database.close();
+  }
+
+  @Override
+  protected void tearDown(final TestDestinationEnv testEnv) {
+    container.close();
+  }
+
+  @Override
+  protected String getImageName() {
+    return "airbyte/source-cockroachdb-strict-encrypt:dev";
+  }
+
+  @Override
+  protected ConnectorSpecification getSpec() throws Exception {
+    return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class));
+  }
+
+  @Override
+  protected JsonNode getConfig() {
+    return config;
+  }
+
+  @Override
+  protected ConfiguredAirbyteCatalog getConfiguredCatalog() {
+    return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(
+        new ConfiguredAirbyteStream()
+            .withSyncMode(SyncMode.INCREMENTAL)
+            .withCursorField(Lists.newArrayList("id"))
+            .withDestinationSyncMode(DestinationSyncMode.APPEND)
+            .withStream(CatalogHelpers.createAirbyteStream(
+                STREAM_NAME,
+                Field.of("id", JsonSchemaPrimitive.NUMBER),
+                Field.of("name", JsonSchemaPrimitive.STRING))
+                .withSupportedSyncModes(
+                    Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))),
+        new ConfiguredAirbyteStream()
+            .withSyncMode(SyncMode.INCREMENTAL)
+            .withCursorField(Lists.newArrayList("id"))
+            .withDestinationSyncMode(DestinationSyncMode.APPEND)
+            .withStream(CatalogHelpers.createAirbyteStream(
+                STREAM_NAME2,
+                Field.of("id", JsonSchemaPrimitive.NUMBER),
+                Field.of("name", JsonSchemaPrimitive.STRING))
+                .withSupportedSyncModes(
+                    Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))));
+  }
+
+  @Override
+  protected List<String> getRegexTests() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  protected JsonNode getState() {
+    return Jsons.jsonNode(new HashMap<>());
+  }
+
+}