From c6cffb2eed2dd6885bc4fe6a4ee2b529722f2ffa Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 4 May 2022 14:49:38 +0300 Subject: [PATCH 1/2] enable DAT for the default clickhouse destination --- .../ClickhouseDestinationAcceptanceTest.java | 35 +++++++++++-------- .../ClickhouseTestDataComparator.java | 25 +++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index a1a16aff1bcb..99f6fef6dd90 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -14,9 +14,9 @@ import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; @@ -48,6 +48,26 @@ protected boolean implementsNamespaces() { return true; } + @Override + protected TestDataComparator getTestDataComparator() { + return new ClickhouseTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + @Override protected String getDefaultSchema(final JsonNode config) { if (config.get("database") == null) { @@ -105,19 +125,6 @@ private List retrieveRecordsFromTable(final String tableName, final St return jdbcDB.queryJsons(query); } - @Override - protected List resolveIdentifier(final String identifier) { - final List result = new ArrayList<>(); - final String resolved = namingResolver.getIdentifier(identifier); - result.add(identifier); - result.add(resolved); - if (!resolved.startsWith("\"")) { - result.add(resolved.toLowerCase()); - result.add(resolved.toUpperCase()); - } - return result; - } - private static JdbcDatabase getDatabase(final JsonNode config) { return Databases.createJdbcDatabase( config.get("username").asText(), diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java new file mode 100644 index 000000000000..df2709993097 --- /dev/null +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java @@ -0,0 +1,25 @@ +package io.airbyte.integrations.destination.clickhouse; + +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; + +import java.util.ArrayList; +import java.util.List; + +public class ClickhouseTestDataComparator extends AdvancedTestDataComparator { + + private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer(); + + @Override + protected List resolveIdentifier(final String identifier) { + final List result = new ArrayList<>(); + final String resolved = namingResolver.getIdentifier(identifier); + result.add(identifier); + result.add(resolved); + if (!resolved.startsWith("\"")) { + result.add(resolved.toLowerCase()); + result.add(resolved.toUpperCase()); + } + return result; + } +} From 72a8e7ce23c7d0ebfff7335a210b6bbf1eb1325c Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 4 May 2022 22:21:54 +0300 Subject: [PATCH 2/2] disable normalization in the tests fully due to broken normalization container for the Clickhouse. --- .../ClickhouseDestinationAcceptanceTest.java | 6 +++- ...shClickhouseDestinationAcceptanceTest.java | 29 ++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 99f6fef6dd90..6dee8276d19e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -35,7 +35,7 @@ protected String getImageName() { @Override protected boolean supportsNormalization() { - return true; + return false; } @Override @@ -188,4 +188,8 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin super.testSyncWithNormalization(messagesFilename, catalogFilename); } + @Disabled + public void specNormalizationValueShouldBeCorrect() throws Exception { + super.specNormalizationValueShouldBeCorrect(); + } } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index a6ab946dfdb7..d5b0707116ec 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -17,6 +17,8 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; + +import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; @@ -42,7 +44,7 @@ protected String getImageName() { @Override protected boolean supportsNormalization() { - return true; + return false; } @Override @@ -55,6 +57,26 @@ protected boolean implementsNamespaces() { return true; } + @Override + protected TestDataComparator getTestDataComparator() { + return new ClickhouseTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + @Override protected String getDefaultSchema(final JsonNode config) { if (config.get("database") == null) { @@ -184,4 +206,9 @@ public void testSyncWithNormalization(final String messagesFilename, final Strin super.testSyncWithNormalization(messagesFilename, catalogFilename); } + @Disabled + public void specNormalizationValueShouldBeCorrect() throws Exception { + super.specNormalizationValueShouldBeCorrect(); + } + }