DAT tearDown should cleanup *all* schemas used (#29312)

* DAT `tearDown` should cleanup *all* schemas used * redshift too * push all method signature changes * Automated Commit - Format and Process Resources Changes * use hashset * TEST_SCHEMAS on setup as well * fix call * Automated Commit - Format and Process Resources Changes --------- Co-authored-by: evantahler <evantahler@users.noreply.github.com>
airbytehq · Aug 11, 2023 · 9f515a8 · 9f515a8
1 parent 3db13e0
commit 9f515a8
Show file tree

Hide file tree

Showing 72 changed files with 257 additions and 164 deletions.
diff --git a/...est/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java b/...est/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java
@@ -22,6 +22,7 @@
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.nio.file.Path;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
@@ -125,7 +126,7 @@ protected List<S3ObjectSummary> getAllSyncedObjects(final String streamName, fin
    * <li>Construct the S3 client.</li>
    */
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     final JsonNode baseConfigJson = getBaseConfigJson();
     // Set a random s3 bucket path for each integration test
     final JsonNode configJson = Jsons.clone(baseConfigJson);
@@ -149,7 +150,7 @@ protected void setup(final TestDestinationEnv testEnv) {
    * Remove all the S3 output from the tests.
    */
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     final List<KeyVersion> keysToDelete = new LinkedList<>();
     final List<S3ObjectSummary> objects = s3Client
         .listObjects(config.getBucketName(), config.getBucketPath())

diff --git a/...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/...main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java
@@ -100,6 +100,8 @@
 
 public abstract class DestinationAcceptanceTest {
 
+  private static final HashSet<String> TEST_SCHEMAS = new HashSet<>();
+
   private static final Random RANDOM = new Random();
   private static final String NORMALIZATION_VERSION = "dev";
 
@@ -210,7 +212,9 @@ protected String getDefaultSchema(final JsonNode config) throws Exception {
     if (config.get("schema") == null) {
       return null;
     }
-    return config.get("schema").asText();
+    final String schema = config.get("schema").asText();
+    TEST_SCHEMAS.add(schema);
+    return schema;
   }
 
   /**
@@ -319,19 +323,21 @@ protected List<JsonNode> retrieveNormalizedRecords(final TestDestinationEnv test
    * postgres database. This function will be called before EACH test.
    *
    * @param testEnv - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
    */
-  protected abstract void setup(TestDestinationEnv testEnv) throws Exception;
+  protected abstract void setup(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws Exception;
 
   /**
    * Function that performs any clean up of external resources required for the test. e.g. delete a
    * postgres database. This function will be called after EACH test. It MUST remove all data in the
    * destination so that there is no contamination across tests.
    *
    * @param testEnv - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
    */
-  protected abstract void tearDown(TestDestinationEnv testEnv) throws Exception;
+  protected abstract void tearDown(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws Exception;
 
   /**
    * @deprecated This method is moved to the AdvancedTestDataComparator. Please move your destination
@@ -354,7 +360,7 @@ void setUpInternal() throws Exception {
     testEnv = new TestDestinationEnv(localRoot);
     mConnectorConfigUpdater = Mockito.mock(ConnectorConfigUpdater.class);
 
-    setup(testEnv);
+    setup(testEnv, TEST_SCHEMAS);
 
     processFactory = new DockerProcessFactory(
         workspaceRoot,
@@ -366,7 +372,7 @@ void setUpInternal() throws Exception {
 
   @AfterEach
   void tearDownInternal() throws Exception {
-    tearDown(testEnv);
+    tearDown(testEnv, TEST_SCHEMAS);
   }
 
   /**
@@ -981,6 +987,7 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception {
     // A randomized namespace is required otherwise you can generate a "false success" with data from a
     // previous run.
     final String namespace = Strings.addRandomSuffix("airbyte_source_namespace", "_", 8);
+    TEST_SCHEMAS.add(namespace);
 
     catalog.getStreams().forEach(stream -> stream.setNamespace(namespace));
     final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(
@@ -1014,10 +1021,12 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception {
             MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())),
             AirbyteCatalog.class);
     final var namespace1 = Strings.addRandomSuffix("sourcenamespace", "_", 8);
+    TEST_SCHEMAS.add(namespace1);
     catalog.getStreams().forEach(stream -> stream.setNamespace(namespace1));
 
     final var diffNamespaceStreams = new ArrayList<AirbyteStream>();
-    final var namespace2 = Strings.addRandomSuffix("diff_sourcenamespace", "_", 8);;
+    final var namespace2 = Strings.addRandomSuffix("diff_sourcenamespace", "_", 8);
+    TEST_SCHEMAS.add(namespace2);
     final var mapper = MoreMappers.initMapper();
     for (final AirbyteStream stream : catalog.getStreams()) {
       final var clonedStream = mapper.readValue(mapper.writeValueAsString(stream),

diff --git a/...ntegrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java b/...ntegrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java
@@ -21,6 +21,7 @@
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -125,7 +126,7 @@ protected boolean supportObjectDataTypeTest() {
    * <li>Construct the Azure Blob client.</li>
    */
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     final JsonNode baseConfigJson = getBaseConfigJson();
 
     configJson = Jsons.jsonNode(ImmutableMap.builder()
@@ -159,7 +160,7 @@ protected void setup(final TestDestinationEnv testEnv) {
    * Remove all the Container output from the tests.
    */
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     final BlobServiceClient storageClient =
         new BlobServiceClientBuilder()
             .endpoint(azureBlobStorageDestinationConfig.getEndpointUrl())

diff --git a/...byte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java b/...byte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java
@@ -45,6 +45,7 @@
 import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.TimeZone;
@@ -203,14 +204,14 @@ protected JsonNode createConfig() throws IOException {
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) throws Exception {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws Exception {
     config = createConfig();
     bigquery = configureBigQuery(config);
     dataset = getBigQueryDataSet(config, bigquery);
   }
 
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     tearDownBigQuery(dataset, bigquery);
   }
 

diff --git a/...va/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/...va/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java
@@ -16,6 +16,7 @@
 import io.airbyte.integrations.destination.record_buffer.FileBuffer;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
 import java.nio.file.Path;
+import java.util.HashSet;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.slf4j.Logger;
@@ -28,15 +29,15 @@ public class BigQueryGcsDestinationAcceptanceTest extends AbstractBigQueryDestin
   private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryGcsDestinationAcceptanceTest.class);
 
   /**
-   * Sets up secretsFile path as well as BigQuery and GCS instances for verification and cleanup This
-   * function will be called before EACH test.
+   * Sets up secretsFile path as well as BigQuery and GCS instances for verification and cleanup This function will be called before EACH test.
    *
-   * @see DestinationAcceptanceTest#setUpInternal()
-   * @param testEnv - information about the test environment.
+   * @param testEnv      - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
+   * @see DestinationAcceptanceTest#setUpInternal()
    */
   @Override
-  protected void setup(TestDestinationEnv testEnv) throws Exception {
+  protected void setup(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws Exception {
     // use secrets file with GCS staging config
     secretsFile = Path.of("secrets/credentials-gcs-staging.json");
     setUpBigQuery();
@@ -52,12 +53,13 @@ protected void setup(TestDestinationEnv testEnv) throws Exception {
   /**
    * Removes data from bigquery and GCS This function will be called after EACH test
    *
-   * @see DestinationAcceptanceTest#tearDownInternal()
-   * @param testEnv - information about the test environment.
+   * @param testEnv      - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
+   * @see DestinationAcceptanceTest#tearDownInternal()
    */
   @Override
-  protected void tearDown(TestDestinationEnv testEnv) {
+  protected void tearDown(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     tearDownBigQuery();
     tearDownGcs();
   }

diff --git a/.../airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java b/.../airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java
@@ -8,6 +8,7 @@
 
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
 import java.nio.file.Path;
+import java.util.HashSet;
 import org.junit.jupiter.api.TestInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -18,28 +19,29 @@ public class BigQueryStandardDestinationAcceptanceTest extends AbstractBigQueryD
   private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryStandardDestinationAcceptanceTest.class);
 
   /**
-   * Sets up secretsFile path and BigQuery instance for verification and cleanup This function will be
-   * called before EACH test.
+   * Sets up secretsFile path and BigQuery instance for verification and cleanup This function will be called before EACH test.
    *
-   * @see DestinationAcceptanceTest#setUpInternal()
-   * @param testEnv - information about the test environment.
+   * @param testEnv      - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
+   * @see DestinationAcceptanceTest#setUpInternal()
    */
   @Override
-  protected void setup(TestDestinationEnv testEnv) throws Exception {
+  protected void setup(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) throws Exception {
     secretsFile = Path.of("secrets/credentials-standard.json");
     setUpBigQuery();
   }
 
   /**
    * Removes data from bigquery This function will be called after EACH test
    *
-   * @see DestinationAcceptanceTest#tearDownInternal()
-   * @param testEnv - information about the test environment.
+   * @param testEnv      - information about the test environment.
+   * @param TEST_SCHEMAS
    * @throws Exception - can throw any exception, test framework will handle.
+   * @see DestinationAcceptanceTest#tearDownInternal()
    */
   @Override
-  protected void tearDown(TestDestinationEnv testEnv) {
+  protected void tearDown(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     tearDownBigQuery();
   }
 

diff --git a/...ava/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java b/...ava/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java
@@ -9,6 +9,7 @@
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
 import io.airbyte.integrations.util.HostPortResolver;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeAll;
@@ -29,7 +30,7 @@ static void initContainer() {
   }
 
   @Override
-  protected void setup(TestDestinationEnv testEnv) {
+  protected void setup(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     configJson = TestDataFactory.createJsonConfig(
         cassandraContainer.getUsername(),
         cassandraContainer.getPassword(),
@@ -41,7 +42,7 @@ protected void setup(TestDestinationEnv testEnv) {
   }
 
   @Override
-  protected void tearDown(TestDestinationEnv testEnv) {
+  protected void tearDown(TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     cassandraCqlProvider.retrieveMetadata().forEach(meta -> {
       var keyspace = meta.value1();
       meta.value2().forEach(table -> cassandraCqlProvider.truncate(keyspace, table));

diff --git a/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/...integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java
@@ -24,6 +24,7 @@
 import java.sql.SQLException;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -164,7 +165,7 @@ protected List<String> resolveIdentifier(final String identifier) {
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test")
         .withFileFromClasspath("Dockerfile", "docker/Dockerfile")
         .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh"))
@@ -184,7 +185,7 @@ protected void setup(final TestDestinationEnv testEnv) {
   }
 
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     db.stop();
     db.close();
   }

diff --git a/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/...a/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java
@@ -23,6 +23,7 @@
 import io.airbyte.integrations.util.HostPortResolver;
 import java.sql.SQLException;
 import java.time.Duration;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -137,15 +138,15 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5")
         .waitingFor(Wait.forHttp("/ping").forPort(8123)
             .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS)));
     db.start();
   }
 
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     db.stop();
     db.close();
   }

diff --git a/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/...o/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java
@@ -20,6 +20,7 @@
 import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider;
 import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -154,14 +155,14 @@ private static JdbcDatabase getDatabase(final JsonNode config) {
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     bastion.initAndStartBastion(network);
     db = (ClickHouseContainer) new ClickHouseContainer("clickhouse/clickhouse-server:22.5").withNetwork(network);
     db.start();
   }
 
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     bastion.stopAndCloseContainers(db);
   }
 

diff --git a/...ntegration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java b/...ntegration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java
@@ -24,6 +24,7 @@
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -118,12 +119,12 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
+  protected void setup(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     // no op
   }
 
   @Override
-  protected void tearDown(final TestDestinationEnv testEnv) {
+  protected void tearDown(final TestDestinationEnv testEnv, HashSet<String> TEST_SCHEMAS) {
     // no op
   }