From d8e9f512d6fa3ba6bb49f1c048a531055c833435 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Fri, 14 Oct 2022 22:23:43 +0200 Subject: [PATCH] Fix multiple breakages in the platform build (#18011) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "🎉 New Destination: bit.io (#15821)" This reverts commit c05135b336e3a8c45dfefca340d0c5b261c7a4f0. * Fix broken acceptance tests --- .../NormalizationRunnerFactory.java | 1 - .../src/main/resources/icons/bitdotio.svg | 40 --- .../seed/destination_definitions.yaml | 7 - .../resources/seed/destination_specs.yaml | 50 ---- airbyte-integrations/builds.md | 1 - .../destination-bitdotio/.dockerignore | 3 - .../destination-bitdotio/Dockerfile | 20 -- .../connectors/destination-bitdotio/README.md | 28 --- .../destination-bitdotio/bootstrap.md | 18 -- .../destination-bitdotio/build.gradle | 25 -- .../postgres/BitDotIoDestination.java | 68 ------ .../BitDotIoDestinationAcceptanceTest.java | 228 ------------------ .../postgres/BitDotIoDestinationTest.java | 22 -- .../src/test/resources/expected_spec.json | 53 ---- .../test/acceptance/BasicAcceptanceTests.java | 2 +- docs/integrations/README.md | 3 +- docs/integrations/destinations/bitdotio.md | 51 ---- 17 files changed, 2 insertions(+), 618 deletions(-) delete mode 100644 airbyte-config/init/src/main/resources/icons/bitdotio.svg delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/.dockerignore delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/Dockerfile delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/README.md delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/bootstrap.md delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/build.gradle delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/src/test/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationTest.java delete mode 100644 airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json delete mode 100644 docs/integrations/destinations/bitdotio.md diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 7ba32daf18d1..cc1530d3fd49 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -30,7 +30,6 @@ public class NormalizationRunnerFactory { .put("airbyte/destination-oracle", ImmutablePair.of("airbyte/normalization-oracle", DestinationType.ORACLE)) .put("airbyte/destination-oracle-strict-encrypt", ImmutablePair.of("airbyte/normalization-oracle", DestinationType.ORACLE)) .put("airbyte/destination-postgres", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) - .put("airbyte/destination-bitdotio", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) .put("airbyte/destination-postgres-strict-encrypt", ImmutablePair.of(BASE_NORMALIZATION_IMAGE_NAME, DestinationType.POSTGRES)) .put("airbyte/destination-redshift", ImmutablePair.of("airbyte/normalization-redshift", DestinationType.REDSHIFT)) .put("airbyte/destination-snowflake", ImmutablePair.of("airbyte/normalization-snowflake", DestinationType.SNOWFLAKE)) diff --git a/airbyte-config/init/src/main/resources/icons/bitdotio.svg b/airbyte-config/init/src/main/resources/icons/bitdotio.svg deleted file mode 100644 index aef56c6946ef..000000000000 --- a/airbyte-config/init/src/main/resources/icons/bitdotio.svg +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 8bfe95fa55d1..71c2d7305fd1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -50,13 +50,6 @@ memory_limit: "1Gi" memory_request: "1Gi" releaseStage: beta -- name: bit.io - destinationDefinitionId: 592af2e0-882c-4ed8-b0fe-a31761146a8d - dockerRepository: airbyte/destination-bitdotio - dockerImageTag: 0.1.0 - documentationUrl: https://docs.airbyte.io/integrations/destinations/bitdotio - icon: bitdotio.svg - releaseStage: alpha - name: Cassandra destinationDefinitionId: 707456df-6f4f-4ced-b5c6-03f73bcad1c5 dockerRepository: airbyte/destination-cassandra diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index bf49a62337d8..0c3794a58c19 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -693,56 +693,6 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bitdotio:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/destinations/bitdotio" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Postgres Destination Spec" - type: "object" - required: - - "database" - - "username" - - "password" - additionalProperties: true - properties: - database: - title: "DB Name" - description: "Name of the database." - type: "string" - order: 2 - schema: - title: "Default Schema" - description: "The default schema tables are written to if the source does\ - \ not specify a namespace. The usual value for this field is \"public\"\ - ." - type: "string" - examples: - - "public" - default: "public" - order: 3 - username: - title: "User" - description: "Username to use to access the database." - type: "string" - order: 4 - password: - title: "Password" - description: "Password associated with the username." - type: "string" - airbyte_secret: true - order: 5 - ssl: "true" - host: "db.bit.io" - ssl_mode: "require" - port: "5432" - supportsIncremental: true - supportsNormalization: true - supportsDBT: true - supported_destination_sync_modes: - - "overwrite" - - "append" - - "append_dedup" - dockerImage: "airbyte/destination-cassandra:0.1.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/cassandra" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 9d72e65aa1f1..7af75b786c27 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -131,7 +131,6 @@ | Azure Blob Storage | [![destination-azure-blob-storage](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-azure-blob-storage%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-azure-blob-storage) | | BigQuery | [![destination-bigquery](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-bigquery%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-bigquery) | | BigQuery Denormalized | [![destination-bigquery-denormalized](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-bigquery-denormalized%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-bigquery-denormalized) | -| bit.io | [![destination-bitdotio](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-bitdotio%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-bitdotio) | | ClickHouse | [![destination-clickhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-clickhouse%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-clickhouse) | | Cassandra | [![destination-cassandra](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-cassandra%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-cassandra) | | Databricks | [![destination-databricks](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-databricks%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-databricks) | diff --git a/airbyte-integrations/connectors/destination-bitdotio/.dockerignore b/airbyte-integrations/connectors/destination-bitdotio/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-bitdotio/Dockerfile b/airbyte-integrations/connectors/destination-bitdotio/Dockerfile deleted file mode 100644 index 8bf1c694a354..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-bitdotio - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-bitdotio - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-bitdotio diff --git a/airbyte-integrations/connectors/destination-bitdotio/README.md b/airbyte-integrations/connectors/destination-bitdotio/README.md deleted file mode 100644 index d3ec6022056b..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/README.md +++ /dev/null @@ -1,28 +0,0 @@ -bit.io Destination -================== -This allows the use of the bit.io service as a destination in Airbyte. - -The bit.io destination is a `SpecModifiyingDestination` that hardcodes the host, port, and -ssl options from the `PostgresDestination`. - -Testing -------- - -To test you need a bit.io user with an empty database. - -Then, create `secrets/credentials.json` - -``` -{ - "username": "", - "database": "", - "connect_password": "" -} -``` - -Then you can run the standard set of gradle-driven tests. - -NOTE THAT THE DATABASE CONTENTS WILL BE DESTROYED BY TESTS. So don't use a database -with anythign in it. - - diff --git a/airbyte-integrations/connectors/destination-bitdotio/bootstrap.md b/airbyte-integrations/connectors/destination-bitdotio/bootstrap.md deleted file mode 100644 index 0f60ccb96299..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/bootstrap.md +++ /dev/null @@ -1,18 +0,0 @@ -# bit.io - -## Overview - -bit.io is a serverless, shareable cloud Postgres database. - -## Endpoints - -This destination connector uses the Postgres protocol. - -## Quick Notes - -- bit.io doesn't support `CREATE DATABASE` - -- This connectors primary change is to hardcode the hostname to `db.bit.io`, the port to `5432`, and the `sslmode` to `require` - - - diff --git a/airbyte-integrations/connectors/destination-bitdotio/build.gradle b/airbyte-integrations/connectors/destination-bitdotio/build.gradle deleted file mode 100644 index 768f5b45ed3b..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.destination.postgres.BitDotIoDestination' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-protocol:protocol-models') - implementation project(':airbyte-integrations:connectors:destination-jdbc') - implementation project(':airbyte-integrations:connectors:destination-postgres') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java b/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java deleted file mode 100644 index 092171ad6ecb..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/src/main/java/io/airbyte/integrations/destination/postgres/BitDotIoDestination.java +++ /dev/null @@ -1,68 +0,0 @@ -/* * Copyright (c) 2022 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.postgres; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode ; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; -import io.airbyte.integrations.destination.postgres.PostgresDestination; -import io.airbyte.protocol.models.ConnectorSpecification; -import java.net.URI; -import java.net.URISyntaxException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BitDotIoDestination extends SpecModifyingDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(BitDotIoDestination.class); - - public BitDotIoDestination() { - super(new PostgresDestination()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - String json = "[ \"database\", \"username\", \"password\"] "; - - ObjectMapper objectMapper = new ObjectMapper(); - - JsonNode jsonNode; - try { - jsonNode = objectMapper.readTree(json); - spec.setDocumentationUrl(new URI("https://docs.airbyte.io/integrations/destinations/bitdotio")); - ((ObjectNode) spec.getConnectionSpecification()).replace("required", jsonNode); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.PORT_LIST_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.HOST_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.PORT_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.JDBC_URL_PARAMS_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_MODE_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).put(JdbcUtils.SSL_KEY, "true"); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).put(JdbcUtils.HOST_KEY, "db.bit.io"); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).put(JdbcUtils.SSL_MODE_KEY, "require"); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).put(JdbcUtils.PORT_KEY, "5432"); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - // This should never happen. - e.printStackTrace(); - } - - return spec; - } - - public static void main(final String[] args) throws Exception { - final Destination destination = new BitDotIoDestination(); - LOGGER.info("starting destination: {}", BitDotIoDestination.class); - new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", BitDotIoDestination.class); - } - -} diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java deleted file mode 100644 index d0b46ece172a..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/src/test-integration/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationAcceptanceTest.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.postgres; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.ExtendedNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.stream.Collectors; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; - -import org.jooq.DSLContext; -import org.jooq.SQLDialect; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BitDotIoDestinationAcceptanceTest extends DestinationAcceptanceTest { - public class BitDotIoConfig { - private String username = ""; - private String database = ""; - private String password = ""; - public String getUsername() { - return username; - } - public String getDatabase() { - return database; - } - public String getPassword() { - return password; - } - public BitDotIoConfig(String username, String database, String password) - { - this.username = username; - this.database = database; - this.password = password; - } - - public String getJdbcUrl() { - String jdbcUrl = ""; - try { - jdbcUrl = "jdbc:postgresql://db.bit.io:5432" + "/" + URLEncoder.encode(database, "UTF-8") + "?sslmode=require"; - } catch (UnsupportedEncodingException e) { - // Should never happen - e.printStackTrace(); - } - return jdbcUrl; - } - } - - - private static final Logger LOGGER = LoggerFactory.getLogger(BitDotIoDestinationAcceptanceTest.class); - private BitDotIoConfig cfg; - private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); - - protected static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - - protected static final String CONFIG_BITIO_USERNAME = "username"; - protected static final String CONFIG_BITIO_DATABASE = "database"; - protected static final String CONFIG_BITIO_CONNECT_PASSWORD = "connect_password"; - - @Override - protected String getImageName() { - return "airbyte/destination-bitdotio:dev"; - } - - @Override - protected JsonNode getConfig() { - - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "db.bit.io") - .put(JdbcUtils.PORT_KEY, 5432) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.USERNAME_KEY, cfg.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, cfg.getPassword()) - .put(JdbcUtils.DATABASE_KEY, cfg.getDatabase()) - .put(JdbcUtils.SSL_KEY, true ) - .put("sslmode", "require" ) - .put("ssl_mode", ImmutableMap.builder().put("mode", "require").build()) - .build()); - } - - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "db.bit.io") - .put(JdbcUtils.PORT_KEY, 5432) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.USERNAME_KEY, cfg.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.DATABASE_KEY, cfg.getDatabase()) - .put(JdbcUtils.SSL_KEY, true) - .put("sslmode", "require" ) - .put("ssl_mode", ImmutableMap.builder().put("mode", "require").build()) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) - .collect(Collectors.toList()); - } - - @Override - protected boolean supportsNormalization() { - return true; - } - - @Override - protected boolean supportsDBT() { - return true; - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); - } - - @Override - protected List resolveIdentifier(final String identifier) { - final List result = new ArrayList<>(); - final String resolved = namingResolver.getIdentifier(identifier); - result.add(identifier); - result.add(resolved); - if (!resolved.startsWith("\"")) { - result.add(resolved.toLowerCase()); - result.add(resolved.toUpperCase()); - } - return result; - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - cfg.getUsername(), - cfg.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - cfg.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query( - ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - } - } - @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a bit.io query credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - - final String fullConfigAsString = Files.readString(CREDENTIALS_PATH); - final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); - final String username = credentialsJson.get(CONFIG_BITIO_USERNAME).asText(); - final String database = credentialsJson.get(CONFIG_BITIO_DATABASE).asText(); - final String password = credentialsJson.get(CONFIG_BITIO_CONNECT_PASSWORD).asText(); - - this.cfg = new BitDotIoConfig(username, database, password) ; - } - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - try (final DSLContext dslContext = DSLContextFactory.create( - cfg.getUsername(), - cfg.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - cfg.getJdbcUrl(), - SQLDialect.POSTGRES)) { - - Database db = new Database(dslContext); - List tables = db.query( - ctx -> ctx - .fetch( "SELECT table_name FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('pg_catalog', 'information_schema');") - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - for (JsonNode node : tables) { - db.query(ctx -> ctx.fetch(String.format("DROP TABLE IF EXISTS %s CASCADE", node.get("table_name")))); - } - List schemas = db.query( - ctx -> ctx - .fetch( "SELECT DISTINCT table_schema FROM information_schema.tables WHERE table_type='BASE TABLE' AND table_schema not IN ('public', 'pg_catalog', 'information_schema');") - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - for (JsonNode node : schemas) { - db.query(ctx -> ctx.fetch(String.format("DROP SCHEMA IF EXISTS %s CASCADE", node.get("table_schema")))); - } - } catch (SQLException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - LOGGER.info("Finished acceptance test for bit.io"); - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/test/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationTest.java b/airbyte-integrations/connectors/destination-bitdotio/src/test/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationTest.java deleted file mode 100644 index d470188acc90..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/src/test/java/io/airbyte/integrations/destination/postgres/BitDotIoDestinationTest.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.postgres; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.ConnectorSpecification; -import org.junit.jupiter.api.Test; - -class BitDotIoDestinationTest { - - @Test - void testGetSpec() throws Exception { - assertEquals(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class), - new BitDotIoDestination().spec()); - } - -} diff --git a/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json deleted file mode 100644 index ca0f9bf4b315..000000000000 --- a/airbyte-integrations/connectors/destination-bitdotio/src/test/resources/expected_spec.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bitdotio", - "supportsIncremental": true, - "supportsNormalization": true, - "supportsDBT": true, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Destination Spec", - "type": "object", - "required": [ - "database", - "username", - "password" - ], - "additionalProperties": true, - "properties": { - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "schema": { - "title": "Default Schema", - "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\".", - "type": "string", - "examples": [ - "public" - ], - "default": "public", - "order": 3 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 4 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "ssl": "true", - "host": "db.bit.io", - "ssl_mode": "require", - "port": "5432" - } - } -} diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index 8dcb24e7064a..4560736b2938 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -925,7 +925,7 @@ void testSyncAfterUpgradeToPerStreamState(final TestInfo testInfo) throws Except // sync one more time. verify it is the equivalent of a full refresh. final String expectedState = - "{\"cursor\":\"6\",\"stream_name\":\"id_and_name\",\"cursor_field\":[\"id\"],\"stream_namespace\":\"public\"}"; + "{\"cursor\":\"6\",\"stream_name\":\"id_and_name\",\"cursor_field\":[\"id\"],\"stream_namespace\":\"public\",\"cursor_record_count\":1}"; LOGGER.info("Starting {} sync 3", testInfo.getDisplayName()); final JobInfoRead connectionSyncRead3 = apiClient.getConnectionApi().syncConnection(new ConnectionIdRequestBody().connectionId(connectionId)); diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 78dfeb98f0b1..fe4fae5cccc1 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -189,8 +189,7 @@ For more information about the grading system, see [Product Release Stages](http | [Amazon Datalake](destinations/aws-datalake.md) | Alpha | No | | [AzureBlobStorage](destinations/azureblobstorage.md) | Alpha | Yes | | [BigQuery](destinations/bigquery.md) | Generally Available | Yes | -| [bit.io](destinations/bitdotio.md) | Alpha | No | -| [Cassandra](destinations/cassandra.md) | Alpha | Yes | +| [Cassandra](destinations/cassandra.md) | Alpha | No | | [Chargify (Keen)](destinations/chargify.md) | Alpha | Yes | | [ClickHouse](destinations/clickhouse.md) | Alpha | Yes | | [Databricks](destinations/databricks.md) | Alpha | No | diff --git a/docs/integrations/destinations/bitdotio.md b/docs/integrations/destinations/bitdotio.md deleted file mode 100644 index 528396500afb..000000000000 --- a/docs/integrations/destinations/bitdotio.md +++ /dev/null @@ -1,51 +0,0 @@ -# bit.io - -This page guides you through the process of setting up the bit.io destination connector. - -The bit.io connector is a modified version of the Postgres connector. - -## Prerequisites - -To use the bit.io destination, you'll need: - -* A bit.io account - -You'll need the following information to configure the bit.io destination: - -* **Username** -* **Connect Password** - Found on your database page on the `Connect` tab (this is NOT your bit.io login password) -* **Database** - The database name, including your username - eg, `adam/FCC`. - - -## Supported sync modes - -The bit.io destination connector supports the -following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Incremental - Deduped History | Yes | | -| Namespaces | Yes | | - -## Schema map - -#### Output Schema - -Each stream will be mapped to a separate table in bit.io. Each table will contain 3 columns: - -* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in - Postgres is `VARCHAR`. -* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. - The column type in Postgres is `TIMESTAMP WITH TIME ZONE`. -* `_airbyte_data`: a json blob representing with the event data. The column type in Postgres - is `JSONB`. - - - -## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------| -| 0.1.0 | 2022-08-20 | | Initial Version