diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java
index 4ad91d5b8f02..a3057cc74450 100644
--- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java
+++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java
@@ -5,38 +5,25 @@
package io.airbyte.test.automaticMigrationAcceptance;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
-import com.google.common.io.Resources;
-import io.airbyte.api.client.generated.ConnectionApi;
import io.airbyte.api.client.generated.DestinationDefinitionApi;
import io.airbyte.api.client.generated.HealthApi;
import io.airbyte.api.client.generated.SourceDefinitionApi;
import io.airbyte.api.client.generated.WorkspaceApi;
import io.airbyte.api.client.invoker.generated.ApiClient;
import io.airbyte.api.client.invoker.generated.ApiException;
-import io.airbyte.api.client.model.generated.ConnectionRead;
-import io.airbyte.api.client.model.generated.ConnectionStatus;
import io.airbyte.api.client.model.generated.DestinationDefinitionRead;
-import io.airbyte.api.client.model.generated.ImportRead;
-import io.airbyte.api.client.model.generated.ImportRead.StatusEnum;
import io.airbyte.api.client.model.generated.SourceDefinitionRead;
import io.airbyte.api.client.model.generated.WorkspaceIdRequestBody;
import io.airbyte.api.client.model.generated.WorkspaceRead;
import io.airbyte.commons.concurrency.VoidCallable;
-import io.airbyte.commons.concurrency.WaitingUtils;
import io.airbyte.commons.resources.MoreResources;
import io.airbyte.commons.util.MoreProperties;
-import io.airbyte.commons.version.AirbyteVersion;
import io.airbyte.test.airbyte_test_container.AirbyteTestContainer;
import java.io.File;
-import java.net.URISyntaxException;
import java.nio.file.Path;
-import java.time.Duration;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -46,18 +33,11 @@
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testcontainers.utility.ComparableVersion;
/**
* This class contains an e2e test simulating what a user encounter when trying to upgrade Airybte.
- *
- * Three invariants are tested:
- *
- * - upgrading pass 0.32.0 without first upgrading to 0.32.0 should error.
- *
- * - upgrading pass 0.32.0 without first upgrading to 0.32.0 should not put the db in a bad state.
- *
- * - upgrading from 0.32.0 to the latest version should work.
+ * - upgrading from 0.32.0 to the latest version should work. - This test previously tested
+ * upgrading from even older versions, which has since been removed
*
* This test runs on the current code version and expects local images with the `dev` tag to be
* available. To do so, run SUB_BUILD=PLATFORM ./gradlew build.
@@ -79,37 +59,22 @@ class MigrationAcceptanceTest {
private static final String TEST_LOCAL_ROOT = "/tmp/airbyte_local_migration_test";
private static final String TEST_LOCAL_DOCKER_MOUNT = "/tmp/airbyte_local_migration_test";
+ private static WorkspaceIdRequestBody workspaceIdRequestBody = null;
+
@Test
@Disabled
void testAutomaticMigration() throws Exception {
- // run version 17 (the oldest version of airbyte that supports auto migration)
- final File version17DockerComposeFile = MoreResources.readResourceAsFile("docker-compose-migration-test-0-17-0-alpha.yaml");
- final Properties version17EnvVariables = MoreProperties
- .envFileToProperties(MoreResources.readResourceAsFile("env-file-migration-test-0-17-0.env"));
- runAirbyte(version17DockerComposeFile, version17EnvVariables, () -> {
- populateDataForFirstRun();
- healthCheck(getApiClient());
- });
-
- LOGGER.info("Finish initial 0.17.0-alpha start..");
-
- // attempt to run from pre-version bump version to post-version bump version. expect failure.
- final File currentDockerComposeFile = MoreResources.readResourceAsFile("docker-compose.yaml");
- // piggybacks off of whatever the existing .env file is, so override default filesystem values in to
- // point at test paths.
- final Properties envFileProperties = overrideDirectoriesForTest(MoreProperties.envFileToProperties(ENV_FILE));
- // use the dev version so the test is run on the current code version.
- envFileProperties.setProperty("VERSION", "dev");
- runAirbyteAndWaitForUpgradeException(currentDockerComposeFile, envFileProperties);
- LOGGER.info("Finished testing upgrade exception..");
-
- // run "faux" major version bump version
+ // start at "faux" major version bump version. This was the last version that required db data
+ // migrations.
final File version32DockerComposeFile = MoreResources.readResourceAsFile("docker-compose-migration-test-0-32-0-alpha.yaml");
-
final Properties version32EnvFileProperties = MoreProperties
.envFileToProperties(MoreResources.readResourceAsFile("env-file-migration-test-0-32-0.env"));
runAirbyte(version32DockerComposeFile, version32EnvFileProperties, MigrationAcceptanceTest::assertHealthy);
+ final File currentDockerComposeFile = MoreResources.readResourceAsFile("docker-compose.yaml");
+ // piggybacks off of whatever the existing .env file is, so override default filesystem values in to
+ // point at test paths.
+ final Properties envFileProperties = overrideDirectoriesForTest(MoreProperties.envFileToProperties(ENV_FILE));
// run from last major version bump to current version.
runAirbyte(currentDockerComposeFile, envFileProperties, MigrationAcceptanceTest::assertHealthy, false);
}
@@ -147,22 +112,6 @@ private void runAirbyte(final File dockerComposeFile,
}
}
- private void runAirbyteAndWaitForUpgradeException(final File dockerComposeFile, final Properties env) throws Exception {
- final WaitForLogLine waitForLogLine = new WaitForLogLine();
- LOGGER.info("Start up Airbyte at version {}", env.get("VERSION"));
- final AirbyteTestContainer airbyteTestContainer = new AirbyteTestContainer.Builder(dockerComposeFile)
- .setEnv(env)
- .setLogListener("bootloader", waitForLogLine.getListener("After that upgrade is complete, you may upgrade to version"))
- .build();
-
- airbyteTestContainer.startAsync();
-
- final Supplier condition = waitForLogLine.hasSeenLine();
- final boolean loggedUpgradeException = WaitingUtils.waitForCondition(Duration.ofSeconds(5), Duration.ofMinutes(1), condition);
- airbyteTestContainer.stopRetainVolumes();
- assertTrue(loggedUpgradeException, "Airbyte failed to throw upgrade exception.");
- }
-
/**
* Allows the test to listen for a specific log line so that the test can end as soon as that log
* line has been encountered.
@@ -191,11 +140,16 @@ private static void assertHealthy() throws ApiException {
assertDataFromApi(apiClient);
}
+ @SuppressWarnings("PMD.NonThreadSafeSingleton")
private static void assertDataFromApi(final ApiClient apiClient) throws ApiException {
- final WorkspaceIdRequestBody workspaceIdRequestBody = assertWorkspaceInformation(apiClient);
+ if (workspaceIdRequestBody != null) {
+ assertEquals(assertWorkspaceInformation(apiClient).getWorkspaceId(), workspaceIdRequestBody.getWorkspaceId());
+ } else {
+ workspaceIdRequestBody = assertWorkspaceInformation(apiClient);
+ }
+
assertSourceDefinitionInformation(apiClient);
assertDestinationDefinitionInformation(apiClient);
- assertConnectionInformation(apiClient, workspaceIdRequestBody);
}
private static void assertSourceDefinitionInformation(final ApiClient apiClient) throws ApiException {
@@ -207,18 +161,9 @@ private static void assertSourceDefinitionInformation(final ApiClient apiClient)
for (final SourceDefinitionRead sourceDefinitionRead : sourceDefinitions) {
if ("435bb9a5-7887-4809-aa58-28c27df0d7ad".equals(sourceDefinitionRead.getSourceDefinitionId().toString())) {
assertEquals(sourceDefinitionRead.getName(), "MySQL");
- assertEquals(sourceDefinitionRead.getDockerImageTag(), "0.2.0");
foundMysqlSourceDefinition = true;
} else if ("decd338e-5647-4c0b-adf4-da0e75f5a750".equals(sourceDefinitionRead.getSourceDefinitionId().toString())) {
- final String[] tagBrokenAsArray = sourceDefinitionRead.getDockerImageTag().replace(".", ",").split(",");
- assertEquals(3, tagBrokenAsArray.length);
- // todo (cgardens) - this is very brittle. depending on when this connector gets updated in
- // source_definitions.yaml this test can start to break. for now just doing quick fix, but we should
- // be able to do an actual version comparison like we do with AirbyteVersion.
- assertTrue(Integer.parseInt(tagBrokenAsArray[0]) >= 0, "actual tag: " + sourceDefinitionRead.getDockerImageTag());
- assertTrue(Integer.parseInt(tagBrokenAsArray[1]) >= 3, "actual tag: " + sourceDefinitionRead.getDockerImageTag());
- assertTrue(Integer.parseInt(tagBrokenAsArray[2]) >= 0, "actual tag: " + sourceDefinitionRead.getDockerImageTag());
- assertTrue(sourceDefinitionRead.getName().contains("Postgres"));
+ assertEquals(sourceDefinitionRead.getName(), "Postgres");
foundPostgresSourceDefinition = true;
}
}
@@ -239,21 +184,11 @@ private static void assertDestinationDefinitionInformation(final ApiClient apiCl
destinationId = destinationDefinitionRead.getDestinationDefinitionId().toString();
if ("25c5221d-dce2-4163-ade9-739ef790f503".equals(destinationId)) {
assertEquals("Postgres", destinationDefinitionRead.getName());
- assertEquals("0.2.0", destinationDefinitionRead.getDockerImageTag());
foundPostgresDestinationDefinition = true;
} else if ("8be1cf83-fde1-477f-a4ad-318d23c9f3c6".equals(destinationId)) {
- final String tag = destinationDefinitionRead.getDockerImageTag();
- final AirbyteVersion currentVersion = new AirbyteVersion(tag);
- final AirbyteVersion previousVersion = new AirbyteVersion("0.2.0");
- final AirbyteVersion finalVersion =
- (currentVersion.checkOnlyPatchVersionIsUpdatedComparedTo(previousVersion) ? currentVersion : previousVersion);
- assertEquals(finalVersion.toString(), currentVersion.toString());
assertTrue(destinationDefinitionRead.getName().contains("Local CSV"));
foundLocalCSVDestinationDefinition = true;
} else if ("424892c4-daac-4491-b35d-c6688ba547ba".equals(destinationId)) {
- final String tag = destinationDefinitionRead.getDockerImageTag();
- final ComparableVersion version = new ComparableVersion(tag);
- assertTrue(version.compareTo(new ComparableVersion("0.3.9")) >= 0);
assertTrue(destinationDefinitionRead.getName().contains("Snowflake"));
foundSnowflakeDestinationDefinition = true;
}
@@ -264,60 +199,17 @@ private static void assertDestinationDefinitionInformation(final ApiClient apiCl
assertTrue(foundSnowflakeDestinationDefinition);
}
- private static void assertConnectionInformation(final ApiClient apiClient, final WorkspaceIdRequestBody workspaceIdRequestBody)
- throws ApiException {
- final ConnectionApi connectionApi = new ConnectionApi(apiClient);
- final List connections = connectionApi.listConnectionsForWorkspace(workspaceIdRequestBody).getConnections();
- assertEquals(connections.size(), 2);
- for (final ConnectionRead connection : connections) {
- if ("a294256f-1abe-4837-925f-91602c7207b4".equals(connection.getConnectionId().toString())) {
- assertEquals("", connection.getPrefix());
- assertEquals("28ffee2b-372a-4f72-9b95-8ed56a8b99c5", connection.getSourceId().toString());
- assertEquals("4e00862d-5484-4f50-9860-f3bbb4317397", connection.getDestinationId().toString());
- assertEquals(ConnectionStatus.ACTIVE, connection.getStatus());
- assertNull(connection.getSchedule());
- } else if ("49dae3f0-158b-4737-b6e4-0eed77d4b74e".equals(connection.getConnectionId().toString())) {
- assertEquals("", connection.getPrefix());
- assertEquals("28ffee2b-372a-4f72-9b95-8ed56a8b99c5", connection.getSourceId().toString());
- assertEquals("5434615d-a3b7-4351-bc6b-a9a695555a30", connection.getDestinationId().toString());
- assertEquals(ConnectionStatus.ACTIVE, connection.getStatus());
- assertNull(connection.getSchedule());
- } else {
- fail("Unknown sync " + connection.getConnectionId().toString());
- }
- }
- }
-
private static WorkspaceIdRequestBody assertWorkspaceInformation(final ApiClient apiClient) throws ApiException {
final WorkspaceApi workspaceApi = new WorkspaceApi(apiClient);
final WorkspaceRead workspace = workspaceApi.listWorkspaces().getWorkspaces().get(0);
- // originally the default workspace started with a hardcoded id. the migration in version 0.29.0
- // took that id and randomized it. we want to check that the id is now NOT that hardcoded id and
- // that all related resources use the updated workspaceId as well.
assertNotNull(workspace.getWorkspaceId().toString());
- assertNotEquals("5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", workspace.getWorkspaceId().toString());
- assertEquals("17f90b72-5ae4-40b7-bc49-d6c2943aea57", workspace.getCustomerId().toString());
- assertEquals("default", workspace.getName());
- assertEquals("default", workspace.getSlug());
- assertEquals(true, workspace.getInitialSetupComplete());
- assertEquals(false, workspace.getAnonymousDataCollection());
- assertEquals(false, workspace.getNews());
- assertEquals(false, workspace.getSecurityUpdates());
- assertEquals(false, workspace.getDisplaySetupWizard());
+ assertNotNull(workspace.getName());
+ assertNotNull(workspace.getSlug());
+ assertEquals(false, workspace.getInitialSetupComplete());
return new WorkspaceIdRequestBody().workspaceId(workspace.getWorkspaceId());
}
- @SuppressWarnings("UnstableApiUsage")
- private static void populateDataForFirstRun() throws ApiException, URISyntaxException {
- final ImportApi deploymentApi = new ImportApi(getApiClient());
- final File file = Path
- .of(Resources.getResource("03a4c904-c91d-447f-ab59-27a43b52c2fd.gz").toURI())
- .toFile();
- final ImportRead importRead = deploymentApi.importArchive(file);
- assertEquals(importRead.getStatus(), StatusEnum.SUCCEEDED);
- }
-
private static void healthCheck(final ApiClient apiClient) {
final HealthApi healthApi = new HealthApi(apiClient);
try {
diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml
deleted file mode 100644
index 116796652765..000000000000
--- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-17-0-alpha.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-#The file is used for testing and is from version 0.17.0-alpha
-version: "3.7"
-#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments
-x-logging: &default-logging
- options:
- max-size: "1m"
- max-file: "1"
- driver: json-file
-services:
- init:
- image: airbyte/init:${VERSION}
- logging: *default-logging
- command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"
- environment:
- - LOCAL_ROOT=${LOCAL_ROOT}
- - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}
- volumes:
- - ${HACK_LOCAL_ROOT_PARENT}:/local_parent
- db:
- image: airbyte/db:${VERSION}
- logging: *default-logging
- restart: unless-stopped
- environment:
- - POSTGRES_USER=${DATABASE_USER}
- - POSTGRES_PASSWORD=${DATABASE_PASSWORD}
- volumes:
- - db:/var/lib/postgresql/data
- seed:
- image: airbyte/seed:${VERSION}
- # Pre-populate the volume if it is empty.
- # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container
- volumes:
- - data:/app/seed
- server:
- image: airbyte/server:${VERSION}
- logging: *default-logging
- restart: unless-stopped
- environment:
- - WEBAPP_URL=${WEBAPP_URL}
- - DATABASE_USER=${DATABASE_USER}
- - DATABASE_PASSWORD=${DATABASE_PASSWORD}
- - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}
- - WORKSPACE_ROOT=${WORKSPACE_ROOT}
- - CONFIG_ROOT=${CONFIG_ROOT}
- - TRACKING_STRATEGY=${TRACKING_STRATEGY}
- - AIRBYTE_VERSION=${VERSION}
- - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
- - TEMPORAL_HOST=${TEMPORAL_HOST}
- ports:
- - 8001:8001
- volumes:
- - workspace:${WORKSPACE_ROOT}
- - data:${CONFIG_ROOT}
-volumes:
- workspace:
- name: ${WORKSPACE_DOCKER_MOUNT}
- data:
- name: ${DATA_DOCKER_MOUNT}
- db:
- name: ${DB_DOCKER_MOUNT}
diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env
deleted file mode 100644
index cdcebcec9b9d..000000000000
--- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-17-0.env
+++ /dev/null
@@ -1,22 +0,0 @@
-VERSION=0.17.0-alpha-db-patch
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
-DATABASE_DB=airbyte
-CONFIG_ROOT=/data
-WORKSPACE_ROOT=/tmp/workspace
-DATA_DOCKER_MOUNT=airbyte_data_migration_test
-DB_DOCKER_MOUNT=airbyte_db_migration_test
-WORKSPACE_DOCKER_MOUNT=airbyte_workspace_migration_test
-LOCAL_ROOT=/tmp/airbyte_local_migration_test
-LOCAL_DOCKER_MOUNT=/tmp/airbyte_local_migration_test
-TRACKING_STRATEGY=logging
-HACK_LOCAL_ROOT_PARENT=/tmp
-WEBAPP_URL=http://localhost:8000/
-API_URL=http://localhost:8001/api/v1/
-TEMPORAL_HOST=airbyte-temporal:7233
-INTERNAL_API_HOST=airbyte-server:8001
-S3_LOG_BUCKET=
-S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-GCP_STORAGE_BUCKET=