From fa2ea5046b8773019dfc2315f428428dfa8a27b1 Mon Sep 17 00:00:00 2001 From: Subodh Kant Chaturvedi Date: Mon, 12 Jul 2021 19:07:10 +0530 Subject: [PATCH] test to show how automatic migration handles deprecated definitions (#4655) * test to show definitions not present in latest seed would be deleted in automatic migration * format * add deprecated config being used scenario --- .../server/migration/RunMigrationTest.java | 46 +++++++++++++------ .../e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json | 14 ++++++ .../4eb22946-2a79-4d20-a3e6-effd234613c3.json | 7 +++ .../d2147be5-fa36-4936-977e-f031affa5895.json | 7 +++ 4 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json create mode 100644 airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/4eb22946-2a79-4d20-a3e6-effd234613c3.json create mode 100644 airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d2147be5-fa36-4936-977e-f031affa5895.json diff --git a/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java b/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java index 412ab6a62963..9d45005655c6 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/migration/RunMigrationTest.java @@ -70,6 +70,8 @@ public class RunMigrationTest { private static final String INITIAL_VERSION = "0.17.0-alpha"; private static final String TARGET_VERSION = Migrations.MIGRATIONS.get(Migrations.MIGRATIONS.size() - 1).getVersion(); + private static final String DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED = "d2147be5-fa36-4936-977e-f031affa5895"; + private static final String DEPRECATED_SOURCE_DEFINITION_BEING_USED = "4eb22946-2a79-4d20-a3e6-effd234613c3"; private List resourceToBeCleanedUp; @BeforeEach @@ -103,7 +105,7 @@ public void testRunMigration() { FileUtils.copyDirectory(dummyDataSource.toFile(), configRoot.toFile()); resourceToBeCleanedUp.add(configRoot.toFile()); final JobPersistence jobPersistence = getJobPersistence(stubAirbyteDB.getDatabase(), file, INITIAL_VERSION); - assertDatabaseVersion(jobPersistence, INITIAL_VERSION); + assertPreMigrationConfigs(configRoot, jobPersistence); runMigration(jobPersistence, configRoot); @@ -115,6 +117,15 @@ public void testRunMigration() { } } + private void assertPreMigrationConfigs(Path configRoot, JobPersistence jobPersistence) throws IOException, JsonValidationException { + assertDatabaseVersion(jobPersistence, INITIAL_VERSION); + ConfigRepository configRepository = new ConfigRepository(FileSystemConfigPersistence.createWithValidation(configRoot)); + Map sourceDefinitionsBeforeMigration = configRepository.listStandardSources().stream() + .collect(Collectors.toMap(c -> c.getSourceDefinitionId().toString(), c -> c)); + assertTrue(sourceDefinitionsBeforeMigration.containsKey(DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED)); + assertTrue(sourceDefinitionsBeforeMigration.containsKey(DEPRECATED_SOURCE_DEFINITION_BEING_USED)); + } + private void assertDatabaseVersion(JobPersistence jobPersistence, String version) throws IOException { final Optional versionFromDb = jobPersistence.getVersion(); assertTrue(versionFromDb.isPresent()); @@ -137,6 +148,12 @@ private void assertSourceDefinitions(ConfigRepository configRepository) throws J .stream() .collect(Collectors.toMap(c -> c.getSourceDefinitionId().toString(), c -> c)); assertTrue(sourceDefinitions.size() >= 59); + // the definition is not present in latest seeds so it should be deleted + assertFalse(sourceDefinitions.containsKey(DEPRECATED_SOURCE_DEFINITION_NOT_BEING_USED)); + // the definition is not present in latest seeds but it was being used as a connection so it should + // not be deleted + assertTrue(sourceDefinitions.containsKey(DEPRECATED_SOURCE_DEFINITION_BEING_USED)); + final StandardSourceDefinition mysqlDefinition = sourceDefinitions.get("435bb9a5-7887-4809-aa58-28c27df0d7ad"); assertEquals("0.2.0", mysqlDefinition.getDockerImageTag()); assertEquals("MySQL", mysqlDefinition.getName()); @@ -210,18 +227,21 @@ private StandardSyncOperation assertSyncOperations(ConfigRepository configReposi } private void assertSources(ConfigRepository configRepository) throws JsonValidationException, IOException { - final List sourceConnections = configRepository.listSourceConnection(); - assertEquals(sourceConnections.size(), 1); - final SourceConnection sourceConnection = sourceConnections.get(0); - assertEquals(sourceConnection.getName(), "MySQL localhost"); - assertEquals(sourceConnection.getSourceDefinitionId().toString(), "435bb9a5-7887-4809-aa58-28c27df0d7ad"); - assertEquals(sourceConnection.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); - assertEquals(sourceConnection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); - assertEquals(sourceConnection.getConfiguration().get("username").asText(), "root"); - assertEquals(sourceConnection.getConfiguration().get("password").asText(), "password"); - assertEquals(sourceConnection.getConfiguration().get("database").asText(), "localhost_test"); - assertEquals(sourceConnection.getConfiguration().get("port").asInt(), 3306); - assertEquals(sourceConnection.getConfiguration().get("host").asText(), "host.docker.internal"); + final Map sources = configRepository.listSourceConnection().stream() + .collect(Collectors.toMap(sourceConnection -> sourceConnection.getSourceId().toString(), sourceConnection -> sourceConnection)); + assertEquals(sources.size(), 2); + final SourceConnection mysqlConnection = sources.get("28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(mysqlConnection.getName(), "MySQL localhost"); + assertEquals(mysqlConnection.getSourceDefinitionId().toString(), "435bb9a5-7887-4809-aa58-28c27df0d7ad"); + assertEquals(mysqlConnection.getWorkspaceId().toString(), "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6"); + assertEquals(mysqlConnection.getSourceId().toString(), "28ffee2b-372a-4f72-9b95-8ed56a8b99c5"); + assertEquals(mysqlConnection.getConfiguration().get("username").asText(), "root"); + assertEquals(mysqlConnection.getConfiguration().get("password").asText(), "password"); + assertEquals(mysqlConnection.getConfiguration().get("database").asText(), "localhost_test"); + assertEquals(mysqlConnection.getConfiguration().get("port").asInt(), 3306); + assertEquals(mysqlConnection.getConfiguration().get("host").asText(), "host.docker.internal"); + assertTrue(sources.containsKey("e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969")); + } private void assertWorkspace(ConfigRepository configRepository) throws JsonValidationException, IOException { diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json b/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json new file mode 100644 index 000000000000..ee2dc444ef37 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/SOURCE_CONNECTION/e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969.json @@ -0,0 +1,14 @@ +{ + "name": "Using a source definition deleted", + "sourceDefinitionId": "4eb22946-2a79-4d20-a3e6-effd234613c3", + "workspaceId": "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6", + "sourceId": "e48cae1a-1f5c-42cc-9ec1-a44ff7fb4969", + "configuration": { + "username": "root", + "password": "password", + "database": "localhost_test", + "port": 3306, + "host": "host.docker.internal" + }, + "tombstone": false +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/4eb22946-2a79-4d20-a3e6-effd234613c3.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/4eb22946-2a79-4d20-a3e6-effd234613c3.json new file mode 100644 index 000000000000..eb083657a0d0 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/4eb22946-2a79-4d20-a3e6-effd234613c3.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "4eb22946-2a79-4d20-a3e6-effd234613c3", + "name": "Old connector still being used", + "dockerRepository": "airbyte/source-mysql", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql" +} diff --git a/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d2147be5-fa36-4936-977e-f031affa5895.json b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d2147be5-fa36-4936-977e-f031affa5895.json new file mode 100644 index 000000000000..03502ee18e91 --- /dev/null +++ b/airbyte-server/src/test/resources/migration/dummy_data/config/STANDARD_SOURCE_DEFINITION/d2147be5-fa36-4936-977e-f031affa5895.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d2147be5-fa36-4936-977e-f031affa5895", + "name": "Old Connector", + "dockerRepository": "airbyte/source-appstore-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-appstore-singer" +}