destination acceptance tests: get normalization and dbt support from …

…definition (#21299)
airbytehq · Jan 18, 2023 · 6c2b9a1 · 6c2b9a1
1 parent 747d15f
commit 6c2b9a1
Show file tree

Hide file tree

Showing 25 changed files with 7 additions and 238 deletions.
diff --git a/...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -285,20 +285,6 @@ protected boolean implementsOverwrite() throws WorkerException {
     }
   }
 
-  /**
-   * Override to return true to if the destination implements basic normalization and it should be
-   * tested here.
-   *
-   * @return - a boolean.
-   */
-  protected boolean supportsNormalization() {
-    return false;
-  }
-
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   /**
    * Override to return true if a destination implements size limits on record size (then destination
    * should redefine getMaxRecordValueLimit() too)
@@ -570,10 +556,8 @@ public void testLineBreakCharacters() throws Exception {
   }
 
   @Test
-  public void specNormalizationValueShouldBeCorrect() throws Exception {
-    final boolean normalizationFromDefinition = normalizationFromDefinition();
-    assertEquals(normalizationFromDefinition, supportsNormalization());
-    if (normalizationFromDefinition) {
+  public void normalizationFromDefinitionValueShouldBeCorrect() {
+    if (normalizationFromDefinition()) {
       boolean normalizationRunnerFactorySupportsDestinationImage;
       try {
         new DefaultNormalizationRunner(
@@ -584,15 +568,10 @@ public void specNormalizationValueShouldBeCorrect() throws Exception {
       } catch (final IllegalStateException e) {
         normalizationRunnerFactorySupportsDestinationImage = false;
       }
-      assertEquals(normalizationFromDefinition, normalizationRunnerFactorySupportsDestinationImage);
+      assertEquals(normalizationFromDefinition(), normalizationRunnerFactorySupportsDestinationImage);
     }
   }
 
-  @Test
-  public void specDBTValueShouldBeCorrect() {
-    assertEquals(dbtFromDefinition(), supportsDBT());
-  }
-
   /**
    * Verify that the integration successfully writes records incrementally. The second run should
    * append records to the datastore instead of overwriting the previous run.
@@ -1781,7 +1760,7 @@ public void testSyncIntegerInfinityDataType() throws Exception {
 
   private void runAndCheck(final AirbyteCatalog catalog, final ConfiguredAirbyteCatalog configuredCatalog, final List<AirbyteMessage> messages)
       throws Exception {
-    if (supportsNormalization()) {
+    if (normalizationFromDefinition()) {
       LOGGER.info("Normalization is supported! Run test with normalization.");
       runAndCheckWithNormalization(messages, configuredCatalog, catalog);
     } else {

diff --git a/.../airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java b/.../airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java
@@ -64,16 +64,6 @@ protected JsonNode getFailCheckConfig() {
     return config;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;
@@ -170,7 +160,7 @@ protected List<JsonNode> retrieveRecordsFromTable(final String tableName, final
   }
 
   protected void setUpBigQuery() throws IOException {
-    //secrets file should be set by the inhereting class
+    // secrets file should be set by the inhereting class
     Assertions.assertNotNull(secretsFile);
     final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8);
     config = BigQueryDestinationTestUtils.createConfig(secretsFile, datasetId);

diff --git a/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java
@@ -66,16 +66,6 @@ protected String getImageName() {
     return "airbyte/destination-clickhouse-strict-encrypt:dev";
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java
@@ -43,16 +43,6 @@ protected String getImageName() {
     return "airbyte/destination-clickhouse:dev";
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java
@@ -48,16 +48,6 @@ protected String getImageName() {
     return "airbyte/destination-clickhouse:dev";
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...ations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/...ations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java
@@ -65,11 +65,6 @@ protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return false;
-  }
-
   @Override
   protected boolean supportBasicDataTypeTest() {
     return true;

diff --git a/...irbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java b/...irbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java
@@ -60,11 +60,6 @@ protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return false;
-  }
-
   @Override
   protected boolean supportBasicDataTypeTest() {
     return true;

diff --git a/...rations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java b/...rations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java
@@ -50,16 +50,6 @@ protected String getImageName() {
     return "airbyte/destination-mssql-strict-encrypt:dev";
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   private JsonNode getConfig(final MSSQLServerContainer<?> db) {
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, db.getHost())

diff --git a/...ration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java b/...ration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java
@@ -39,16 +39,6 @@ protected String getImageName() {
     return "airbyte/destination-mssql:dev";
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   private JsonNode getConfig(final MSSQLServerContainer<?> db) {
     return Jsons.jsonNode(ImmutableMap.builder()
         .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db))

diff --git a/...ion/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java b/...ion/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java
@@ -42,16 +42,6 @@ protected String getImageName() {
     return "airbyte/destination-mssql:dev";
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   private JsonNode getConfig(final MSSQLServerContainer<?> db) {
 
     return Jsons.jsonNode(ImmutableMap.builder()

diff --git a/...ion/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java b/...ion/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java
@@ -77,21 +77,11 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   private static Database getDatabaseFromConfig(final JsonNode config) {
     final DSLContext dslContext = DSLContextFactory.create(
         config.get(JdbcUtils.USERNAME_KEY).asText(),

diff --git a/...o/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java
@@ -44,21 +44,12 @@ protected String getImageName() {
     return "airbyte/destination-mysql-strict-encrypt:dev";
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
 
   @Override
   protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   @Override
   protected TestDataComparator getTestDataComparator() {
     return new MySqlTestDataComparator();

diff --git a/...ration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/...ration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java
@@ -50,21 +50,12 @@ protected String getImageName() {
     return "airbyte/destination-mysql:dev";
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
 
   @Override
   protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
   @Override
   protected TestDataComparator getTestDataComparator() {
     return new MySqlTestDataComparator();

diff --git a/...ion/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java b/...ion/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java
@@ -71,16 +71,6 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...tions/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/...tions/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java
@@ -81,11 +81,6 @@ protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv env,
                                                      final String streamName,

diff --git a/...n/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/...n/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java
@@ -183,11 +183,6 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
     sshBastionContainer.stopAndCloseContainers(db);
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...o/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java
@@ -83,11 +83,6 @@ protected boolean implementsNamespaces() {
     return true;
   }
 
-  @Override
-  protected boolean supportsDBT() {
-    return false;
-  }
-
   @Override
   protected TestDataComparator getTestDataComparator() {
     return new OracleTestDataComparator();

diff --git a/...yte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/...yte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java
@@ -90,16 +90,6 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/.../java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java b/.../java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java
@@ -71,16 +71,6 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...ntegrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java b/...ntegrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java
@@ -82,16 +82,6 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
-
   @Override
   protected boolean implementsNamespaces() {
     return true;

diff --git a/...va/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java b/...va/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java
@@ -71,15 +71,6 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
         .collect(Collectors.toList());
   }
 
-  @Override
-  protected boolean supportsNormalization() {
-    return true;
-  }
-
-  @Override
-  protected boolean supportsDBT() {
-    return true;
-  }
 
   @Override
   protected boolean implementsNamespaces() {