From 48b2d4cbc9be8c99fbf238194d743cc6f177d93f Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Thu, 8 Oct 2020 19:31:44 -0700 Subject: [PATCH 01/12] add docker & doc info to connector definitions --- .../22f6c74f-5699-40ff-833c-4a879ea40133.json | 5 ++++- .../25c5221d-dce2-4163-ade9-739ef790f503.json | 5 ++++- .../8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json | 5 ++++- .../9fed261d-d107-47fd-8c8b-323023db6e20.json | 5 ++++- .../decd338e-5647-4c0b-adf4-da0e75f5a750.json | 5 ++++- .../e094cb9a-26de-4645-8761-65c0c425d1de.json | 5 ++++- .../src/main/resources/types/StandardDestination.yaml | 9 +++++++++ .../models/src/main/resources/types/StandardSource.yaml | 9 +++++++++ 8 files changed, 42 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/22f6c74f-5699-40ff-833c-4a879ea40133.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/22f6c74f-5699-40ff-833c-4a879ea40133.json index c5f3f2acec61..a21302ea0932 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/22f6c74f-5699-40ff-833c-4a879ea40133.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/22f6c74f-5699-40ff-833c-4a879ea40133.json @@ -1,4 +1,7 @@ { "destinationId": "22f6c74f-5699-40ff-833c-4a879ea40133", - "name": "BigQuery" + "name": "BigQuery", + "dockerRepository": "airbyte/integration-singer-bigquery-destination", + "dockerImageTag": "0.1.4", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-bigquery-destination" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/25c5221d-dce2-4163-ade9-739ef790f503.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/25c5221d-dce2-4163-ade9-739ef790f503.json index 50a69bc2b60f..91f29259a944 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/25c5221d-dce2-4163-ade9-739ef790f503.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/25c5221d-dce2-4163-ade9-739ef790f503.json @@ -1,4 +1,7 @@ { "destinationId": "25c5221d-dce2-4163-ade9-739ef790f503", - "name": "Postgres" + "name": "Postgres", + "dockerRepository": "airbyte/integration-singer-postgres-destination", + "dockerImageTag": "0.1.2", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-postgres-destination" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json index bd7504517bfa..c5318d755780 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION/8be1cf83-fde1-477f-a4ad-318d23c9f3c6.json @@ -1,4 +1,7 @@ { "destinationId": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6", - "name": "Local CSV" + "name": "Local CSV", + "dockerRepository": "airbyte/integration-singer-csv-destination", + "dockerImageTag": "0.1.1", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-csv-destination" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/9fed261d-d107-47fd-8c8b-323023db6e20.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/9fed261d-d107-47fd-8c8b-323023db6e20.json index d42eecdc45b2..e1c48df59e0f 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/9fed261d-d107-47fd-8c8b-323023db6e20.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/9fed261d-d107-47fd-8c8b-323023db6e20.json @@ -1,4 +1,7 @@ { "sourceId": "9fed261d-d107-47fd-8c8b-323023db6e20", - "name": "exchangeratesapi.io" + "name": "exchangeratesapi.io", + "dockerRepository": "airbyte/integration-singer-exchangerateapi_io-source", + "dockerImageTag": "0.1.3", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-exchangeratesapi_io-source" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/decd338e-5647-4c0b-adf4-da0e75f5a750.json index 7b2ec5dff382..8e7a6939e5e3 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/decd338e-5647-4c0b-adf4-da0e75f5a750.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/decd338e-5647-4c0b-adf4-da0e75f5a750.json @@ -1,4 +1,7 @@ { "sourceId": "decd338e-5647-4c0b-adf4-da0e75f5a750", - "name": "Postgres" + "name": "Postgres", + "dockerRepository": "airbyte/integration-singer-postgres-source", + "dockerImageTag": "0.1.3", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-postgres-source" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/e094cb9a-26de-4645-8761-65c0c425d1de.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/e094cb9a-26de-4645-8761-65c0c425d1de.json index f40c40b2d614..edda93d3f894 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/e094cb9a-26de-4645-8761-65c0c425d1de.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/e094cb9a-26de-4645-8761-65c0c425d1de.json @@ -1,4 +1,7 @@ { "sourceId": "e094cb9a-26de-4645-8761-65c0c425d1de", - "name": "Stripe" + "name": "Stripe", + "dockerRepository": "airbyte/integration-singer-stripe-source", + "dockerImageTag": "0.1.2", + "documentationUrl": "https://hub.docker.com/r/airbyte/integration-singer-stripe-source" } diff --git a/airbyte-config/models/src/main/resources/types/StandardDestination.yaml b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml index 5c584dc6a255..0dc8dcf2082d 100644 --- a/airbyte-config/models/src/main/resources/types/StandardDestination.yaml +++ b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml @@ -7,6 +7,9 @@ type: object required: - destinationId - name + - dockerRepository + - dockerImageTag + - documentationUrl additionalProperties: false properties: destinationId: @@ -14,3 +17,9 @@ properties: format: uuid name: type: string + dockerRepository: + type: string + dockerImageTag: + type: string + documentationUrl: + type: string diff --git a/airbyte-config/models/src/main/resources/types/StandardSource.yaml b/airbyte-config/models/src/main/resources/types/StandardSource.yaml index cfca7f8dd1c5..708d552c5e9b 100644 --- a/airbyte-config/models/src/main/resources/types/StandardSource.yaml +++ b/airbyte-config/models/src/main/resources/types/StandardSource.yaml @@ -7,6 +7,9 @@ type: object required: - sourceId - name + - dockerRepository + - dockerImageTag + - documentationUrl additionalProperties: false properties: sourceId: @@ -14,3 +17,9 @@ properties: format: uuid name: type: string + dockerRepository: + type: string + dockerImageTag: + type: string + documentationUrl: + type: string From 46151b76a62b969841ea437e3a6451dfda764a36 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Thu, 8 Oct 2020 20:54:11 -0700 Subject: [PATCH 02/12] save --- airbyte-api/src/main/openapi/config.yaml | 147 ++++++++++++++---- .../io/airbyte/commons/docker/DockerUtil.java | 7 + .../commons/docker/DockerUtilTest.java | 15 ++ .../DestinationConnectionImplementation.yaml | 4 +- .../types/SourceConnectionImplementation.yaml | 4 +- .../job_factory/DefaultSyncJobFactory.java | 25 ++- .../DefaultSchedulerPersistence.java | 26 ++-- .../persistence/SchedulerPersistence.java | 10 +- .../DefaultSchedulerPersistenceTest.java | 94 +++++++---- .../server/handlers/SchedulerHandler.java | 41 +++-- .../SourceImplementationsHandler.java | 7 +- .../server/handlers/SchedulerHandlerTest.java | 18 ++- .../io/airbyte/workers/TestConfigHelpers.java | 8 +- 13 files changed, 295 insertions(+), 111 deletions(-) create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java create mode 100644 airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 57497ccd250d..fdcb41e45a8a 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -121,6 +121,48 @@ paths: description: Workspace not found "422": $ref: "#/components/responses/InvalidInput" + /v1/sources/create: + post: + tags: + - source + summary: Creates a source + operationId: createSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceCreate" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceRead" + "422": + $ref: "#/components/responses/InvalidInput" + /v1/sources/update: + post: + tags: + - source + summary: Update a source + operationId: updateSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceUpdate" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceRead" + "404": + description: Source not found + "422": + $ref: "#/components/responses/InvalidInput" /v1/sources/list: post: tags: @@ -178,7 +220,7 @@ paths: schema: $ref: "#/components/schemas/SourceSpecificationRead" "404": - description: Source Specification not found + description: Source not found "422": $ref: "#/components/responses/InvalidInput" /v1/source_implementations/create: @@ -337,6 +379,29 @@ paths: description: Source Implementation not found "422": $ref: "#/components/responses/InvalidInput" + /v1/destinations/update: + post: + tags: + - destination + summary: Update destination + operationId: updateDestination + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationUpdate" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationRead" + "404": + description: Destination not found + "422": + $ref: "#/components/responses/InvalidInput" /v1/destinations/list: post: tags: @@ -875,16 +940,46 @@ components: properties: sourceId: $ref: "#/components/schemas/SourceId" + SourceCreate: + type: object + required: + - name + - defaultDockerRepository + - defaultDockerImageVersion + properties: + name: + type: string + defaultDockerRepository: + type: string + defaultDockerImageVersion: + type: string + SourceUpdate: + type: object + description: Update the source. Currently, the only allowed attribute to update is the default docker image version. + required: + - sourceId + - defaultDockerImageVersion + properties: + sourceId: + $ref: "#/components/schemas/SourceId" + defaultDockerImageVersion: + type: string SourceRead: type: object required: - sourceId - name + - defaultDockerRepository + - defaultDockerImageVersion properties: sourceId: $ref: "#/components/schemas/SourceId" name: type: string + defaultDockerRepository: + type: string + defaultDockerImageVersion: + type: string SourceReadList: type: object required: @@ -895,21 +990,15 @@ components: items: $ref: "#/components/schemas/SourceRead" # SOURCE SPECIFICATION - SourceSpecificationId: - type: string - format: uuid SourceSpecification: description: The specification for what values are required to configure the source. example: { user: { type: string } } SourceSpecificationRead: type: object required: - - sourceSpecificationId - sourceId - specification properties: - sourceSpecificationId: - $ref: "#/components/schemas/SourceSpecificationId" sourceId: $ref: "#/components/schemas/SourceId" documentationUrl: @@ -934,14 +1023,14 @@ components: type: object required: - workspaceId - - sourceSpecificationId + - sourceId - connectionConfiguration - name properties: workspaceId: $ref: "#/components/schemas/WorkspaceId" - sourceSpecificationId: - $ref: "#/components/schemas/SourceSpecificationId" + sourceId: + $ref: "#/components/schemas/SourceId" connectionConfiguration: $ref: "#/components/schemas/SourceConfiguration" name: @@ -965,7 +1054,6 @@ components: - sourceId - sourceImplementationId - workspaceId - - sourceSpecificationId - connectionConfiguration - name - sourceName @@ -976,8 +1064,6 @@ components: $ref: "#/components/schemas/SourceImplementationId" workspaceId: $ref: "#/components/schemas/WorkspaceId" - sourceSpecificationId: - $ref: "#/components/schemas/SourceSpecificationId" connectionConfiguration: $ref: "#/components/schemas/SourceConfiguration" name: @@ -1011,16 +1097,32 @@ components: properties: destinationId: $ref: "#/components/schemas/DestinationId" + DestinationUpdate: + type: object + required: + - destinationId + - destinationImageVersion + properties: + destinationId: + $ref: "#/components/schemas/DestinationId" + defaultDockerImageVersion: + type: string DestinationRead: type: object required: - destinationId - name + - defaultDockerRepository + - defaultDockerImageVersion properties: destinationId: $ref: "#/components/schemas/DestinationId" name: type: string + defaultDockerRepository: + type: string + defaultDockerImageVersion: + type: string DestinationReadList: type: object required: @@ -1031,21 +1133,15 @@ components: items: $ref: "#/components/schemas/DestinationRead" # DESTINATION SPECIFICATION - DestinationSpecificationId: - type: string - format: uuid DestinationSpecification: description: The specification for what values are required to configure the destination. example: { user: { type: string } } DestinationSpecificationRead: type: object required: - - destinationSpecificationId - destinationId - specification properties: - destinationSpecificationId: - $ref: "#/components/schemas/DestinationSpecificationId" destinationId: $ref: "#/components/schemas/DestinationId" documentationUrl: @@ -1070,13 +1166,13 @@ components: type: object required: - workspaceId - - destinationSpecificationId + - destinationId - connectionConfiguration properties: workspaceId: $ref: "#/components/schemas/WorkspaceId" - destinationSpecificationId: - $ref: "#/components/schemas/DestinationSpecificationId" + destinationId: + $ref: "#/components/schemas/DestinationId" connectionConfiguration: $ref: "#/components/schemas/DestinationConfiguration" name: @@ -1100,7 +1196,6 @@ components: - destinationId - destinationImplementationId - workspaceId - - destinationSpecificationId - connectionConfiguration - name - destinationName @@ -1111,8 +1206,6 @@ components: $ref: "#/components/schemas/DestinationImplementationId" workspaceId: $ref: "#/components/schemas/WorkspaceId" - destinationSpecificationId: - $ref: "#/components/schemas/DestinationSpecificationId" connectionConfiguration: $ref: "#/components/schemas/DestinationConfiguration" name: @@ -1308,7 +1401,6 @@ components: - check_connection_source - check_connection_destination - discover_schema - - get_spec - sync JobListRequestBody: type: object @@ -1420,7 +1512,6 @@ components: - syncSchema - status - source - - isSyncing properties: connectionId: $ref: "#/components/schemas/ConnectionId" @@ -1447,8 +1538,6 @@ components: description: epoch time of last sync. null if no sync has taken place. type: integer format: int64 - isSyncing: - type: boolean WbConnectionReadList: type: object required: diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java new file mode 100644 index 000000000000..5b6e339b5c79 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java @@ -0,0 +1,7 @@ +package io.airbyte.commons.docker; + +public class DockerUtil { + public static String getTaggedImageName(String dockerRepository, String tag){ + return String.join(":", dockerRepository, tag); + } +} diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java new file mode 100644 index 000000000000..f0c3ade13193 --- /dev/null +++ b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java @@ -0,0 +1,15 @@ +package io.airbyte.commons.docker; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DockerUtilTest { + + @Test + void testGetTaggedImageName() { + String repository = "airbyte/repo"; + String tag = "12.3"; + assertEquals("airbyte/repo:12.3", DockerUtil.getTaggedImageName(repository, tag)); + } +} diff --git a/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml b/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml index bf2aa53caa4e..cfaa6259e3d4 100644 --- a/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml +++ b/airbyte-config/models/src/main/resources/types/DestinationConnectionImplementation.yaml @@ -6,7 +6,7 @@ description: information required for connection to a destination. type: object required: - name - - destinationSpecificationId + - destinationId - workspaceId - destinationImplementationId - configuration @@ -15,7 +15,7 @@ additionalProperties: false properties: name: type: string - destinationSpecificationId: + destinationId: type: string format: uuid workspaceId: diff --git a/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml b/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml index c282aba7fac1..7cb130bd9b76 100644 --- a/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml +++ b/airbyte-config/models/src/main/resources/types/SourceConnectionImplementation.yaml @@ -6,7 +6,7 @@ description: information required for connection to a destination. type: object required: - name - - sourceSpecificationId + - sourceId - sourceImplementationId - workspaceId - configuration @@ -15,7 +15,7 @@ additionalProperties: false properties: name: type: string - sourceSpecificationId: + sourceId: type: string format: uuid workspaceId: diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java index 6bcaccd72e61..e18528986219 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java @@ -24,11 +24,17 @@ package io.airbyte.scheduler.job_factory; +import io.airbyte.commons.docker.DockerUtil; import io.airbyte.commons.json.JsonValidationException; +import io.airbyte.config.DestinationConnectionImplementation; +import io.airbyte.config.SourceConnectionImplementation; +import io.airbyte.config.StandardDestination; +import io.airbyte.config.StandardSource; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.scheduler.persistence.SchedulerPersistence; + import java.io.IOException; import java.util.UUID; @@ -47,11 +53,24 @@ public DefaultSyncJobFactory(final SchedulerPersistence schedulerPersistence, public Long create(final UUID connectionId) { try { final StandardSync standardSync = configRepository.getStandardSync(connectionId); + final SourceConnectionImplementation sourceConnectionImplementation = + configRepository.getSourceConnectionImplementation(standardSync.getSourceImplementationId()); + final DestinationConnectionImplementation destinationConnectionImplementation = + configRepository.getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()); + + final StandardSource source = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); + final StandardDestination destination = configRepository.getStandardDestination(destinationConnectionImplementation.getDestinationId()); + + final String sourceImageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String destinationImageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); return schedulerPersistence.createSyncJob( - configRepository.getSourceConnectionImplementation(standardSync.getSourceImplementationId()), - configRepository.getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()), - standardSync); + sourceConnectionImplementation, + destinationConnectionImplementation, + standardSync, + sourceImageName, + destinationImageName); + } catch (IOException | JsonValidationException | ConfigNotFoundException e) { throw new RuntimeException(e); } diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java index 8d49c766541b..c2019e45974b 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java @@ -73,7 +73,7 @@ public DefaultSchedulerPersistence(BasicDataSource connectionPool) { } @Override - public long createSourceCheckConnectionJob(SourceConnectionImplementation sourceImplementation) throws IOException { + public long createSourceCheckConnectionJob(SourceConnectionImplementation sourceImplementation, String dockerImageName) throws IOException { final String scope = ScopeHelper.createScope( JobConfig.ConfigType.CHECK_CONNECTION_SOURCE, @@ -81,8 +81,7 @@ public long createSourceCheckConnectionJob(SourceConnectionImplementation source final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(sourceImplementation.getConfiguration()) - .withDockerImage(Integrations.findBySpecId(sourceImplementation.getSourceSpecificationId()) - .getTaggedImage()); + .withDockerImage(dockerImageName); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.CHECK_CONNECTION_SOURCE) @@ -92,7 +91,7 @@ public long createSourceCheckConnectionJob(SourceConnectionImplementation source } @Override - public long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation) throws IOException { + public long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation, String dockerImageName) throws IOException { final String scope = ScopeHelper.createScope( JobConfig.ConfigType.CHECK_CONNECTION_DESTINATION, @@ -100,8 +99,7 @@ public long createDestinationCheckConnectionJob(DestinationConnectionImplementat final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(destinationImplementation.getConfiguration()) - .withDockerImage(Integrations.findBySpecId(destinationImplementation.getDestinationSpecificationId()) - .getTaggedImage()); + .withDockerImage(dockerImageName); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.CHECK_CONNECTION_DESTINATION) @@ -111,17 +109,15 @@ public long createDestinationCheckConnectionJob(DestinationConnectionImplementat } @Override - public long createDiscoverSchemaJob(SourceConnectionImplementation sourceImplementation) throws IOException { + public long createDiscoverSchemaJob(SourceConnectionImplementation sourceImplementation, String dockerImageName) throws IOException { final String scope = ScopeHelper.createScope( JobConfig.ConfigType.DISCOVER_SCHEMA, sourceImplementation.getSourceImplementationId().toString()); - final String imageName = Integrations.findBySpecId(sourceImplementation.getSourceSpecificationId()) - .getTaggedImage(); final JobDiscoverSchemaConfig jobDiscoverSchemaConfig = new JobDiscoverSchemaConfig() .withConnectionConfiguration(sourceImplementation.getConfiguration()) - .withDockerImage(imageName); + .withDockerImage(dockerImageName); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.DISCOVER_SCHEMA) @@ -133,19 +129,19 @@ public long createDiscoverSchemaJob(SourceConnectionImplementation sourceImpleme @Override public long createSyncJob(SourceConnectionImplementation sourceImplementation, DestinationConnectionImplementation destinationImplementation, - StandardSync standardSync) + StandardSync standardSync, + String sourceDockerImageName, + String destinationDockerImageName) throws IOException { final UUID connectionId = standardSync.getConnectionId(); final String scope = ScopeHelper.createScope(JobConfig.ConfigType.SYNC, connectionId.toString()); - final String sourceImageName = Integrations.findBySpecId(sourceImplementation.getSourceSpecificationId()).getTaggedImage(); - final String destinationImageName = Integrations.findBySpecId(destinationImplementation.getDestinationSpecificationId()).getTaggedImage(); final JobSyncConfig jobSyncConfig = new JobSyncConfig() .withSourceConnectionImplementation(sourceImplementation) - .withSourceDockerImage(sourceImageName) + .withSourceDockerImage(sourceDockerImageName) .withDestinationConnectionImplementation(destinationImplementation) - .withDestinationDockerImage(destinationImageName) + .withDestinationDockerImage(destinationDockerImageName) .withStandardSync(standardSync); final Optional previousJobOptional = getLastSyncJob(connectionId); diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/SchedulerPersistence.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/SchedulerPersistence.java index cd498541f52f..37c292c62d71 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/SchedulerPersistence.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/SchedulerPersistence.java @@ -38,15 +38,17 @@ public interface SchedulerPersistence { - long createSourceCheckConnectionJob(SourceConnectionImplementation sourceImplementation) throws IOException; + long createSourceCheckConnectionJob(SourceConnectionImplementation sourceImplementation, String dockerImage) throws IOException; - long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation) throws IOException; + long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation, String dockerImage) throws IOException; - long createDiscoverSchemaJob(SourceConnectionImplementation sourceImplementation) throws IOException; + long createDiscoverSchemaJob(SourceConnectionImplementation sourceImplementation, String dockerImage) throws IOException; long createSyncJob(SourceConnectionImplementation sourceImplementation, DestinationConnectionImplementation destinationImplementation, - StandardSync standardSync) + StandardSync standardSync, + String sourceDockerImage, + String destinationDockerImage) throws IOException; Job getJob(long jobId) throws IOException; diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java index a138031ea047..cde68a26c383 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java @@ -52,6 +52,7 @@ import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.ScopeHelper; + import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; @@ -62,6 +63,7 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; + import org.apache.commons.dbcp2.BasicDataSource; import org.jooq.Record; import org.junit.jupiter.api.AfterAll; @@ -77,6 +79,8 @@ class DefaultSchedulerPersistenceTest { private static PostgreSQLContainer container; private static BasicDataSource connectionPool; + private static final String SOURCE_IMAGE_NAME = "daxtarity/sourceimagename"; + private static final String DESTINATION_IMAGE_NAME = "daxtarity/destinationimagename"; private static final SourceConnectionImplementation SOURCE_CONNECTION_IMPLEMENTATION; private static final DestinationConnectionImplementation DESTINATION_CONNECTION_IMPLEMENTATION; private static final StandardSync STANDARD_SYNC; @@ -186,14 +190,13 @@ private Record getJobRecord(long jobId) throws SQLException { @Test public void testCreateSourceCheckConnectionJob() throws IOException, SQLException { - final long jobId = schedulerPersistence.createSourceCheckConnectionJob(SOURCE_CONNECTION_IMPLEMENTATION); + final long jobId = schedulerPersistence.createSourceCheckConnectionJob(SOURCE_CONNECTION_IMPLEMENTATION, SOURCE_IMAGE_NAME); final Record jobEntry = getJobRecord(jobId); - final String imageName = Integrations.findBySpecId(SOURCE_CONNECTION_IMPLEMENTATION.getSourceSpecificationId()).getTaggedImage(); final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(SOURCE_CONNECTION_IMPLEMENTATION.getConfiguration()) - .withDockerImage(imageName); + .withDockerImage(SOURCE_IMAGE_NAME); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.CHECK_CONNECTION_SOURCE) @@ -204,15 +207,13 @@ public void testCreateSourceCheckConnectionJob() throws IOException, SQLExceptio @Test public void testCreateDestinationCheckConnectionJob() throws IOException, SQLException { - final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(DESTINATION_CONNECTION_IMPLEMENTATION); + final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(DESTINATION_CONNECTION_IMPLEMENTATION, DESTINATION_IMAGE_NAME); final Record jobEntry = getJobRecord(jobId); - final String imageName = Integrations.findBySpecId(DESTINATION_CONNECTION_IMPLEMENTATION.getDestinationSpecificationId()) - .getTaggedImage(); final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(DESTINATION_CONNECTION_IMPLEMENTATION.getConfiguration()) - .withDockerImage(imageName); + .withDockerImage(DESTINATION_IMAGE_NAME); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.CHECK_CONNECTION_DESTINATION) @@ -223,14 +224,13 @@ public void testCreateDestinationCheckConnectionJob() throws IOException, SQLExc @Test public void testCreateDiscoverSchemaJob() throws IOException, SQLException { - final long jobId = schedulerPersistence.createDiscoverSchemaJob(SOURCE_CONNECTION_IMPLEMENTATION); + final long jobId = schedulerPersistence.createDiscoverSchemaJob(SOURCE_CONNECTION_IMPLEMENTATION, SOURCE_IMAGE_NAME); final Record jobEntry = getJobRecord(jobId); - final String imageName = Integrations.findBySpecId(SOURCE_CONNECTION_IMPLEMENTATION.getSourceSpecificationId()).getTaggedImage(); final JobDiscoverSchemaConfig jobDiscoverSchemaConfig = new JobDiscoverSchemaConfig() .withConnectionConfiguration(SOURCE_CONNECTION_IMPLEMENTATION.getConfiguration()) - .withDockerImage(imageName); + .withDockerImage(SOURCE_IMAGE_NAME); final JobConfig jobConfig = new JobConfig() .withConfigType(JobConfig.ConfigType.DISCOVER_SCHEMA) @@ -244,18 +244,17 @@ public void testCreateSyncJob() throws IOException, SQLException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Record jobEntry = getJobRecord(jobId); - final String sourceImageName = Integrations.findBySpecId(SOURCE_CONNECTION_IMPLEMENTATION.getSourceSpecificationId()).getTaggedImage(); - final String destinationImageName = Integrations.findBySpecId(DESTINATION_CONNECTION_IMPLEMENTATION.getDestinationSpecificationId()) - .getTaggedImage(); final JobSyncConfig jobSyncConfig = new JobSyncConfig() .withSourceConnectionImplementation(SOURCE_CONNECTION_IMPLEMENTATION) - .withSourceDockerImage(sourceImageName) + .withSourceDockerImage(SOURCE_IMAGE_NAME) .withDestinationConnectionImplementation(DESTINATION_CONNECTION_IMPLEMENTATION) - .withDestinationDockerImage(destinationImageName) + .withDestinationDockerImage(DESTINATION_IMAGE_NAME) .withStandardSync(STANDARD_SYNC); final JobConfig jobConfig = new JobConfig() @@ -270,7 +269,9 @@ public void testGetJob() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); when(timeSupplier.get()).thenReturn(NOW); @@ -286,7 +287,9 @@ void testUpdateStatus() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Job created = schedulerPersistence.getJob(jobId); @@ -304,7 +307,9 @@ void testUpdateLogPath() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Job created = schedulerPersistence.getJob(jobId); @@ -322,7 +327,9 @@ void testWriteOutput() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); when(timeSupplier.get()).thenReturn(Instant.ofEpochMilli(4242)); final JobOutput jobOutput = new JobOutput().withOutputType(JobOutput.OutputType.DISCOVER_SCHEMA); @@ -338,7 +345,9 @@ public void testListJobs() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final List actualList = schedulerPersistence .listJobs(JobConfig.ConfigType.SYNC, STANDARD_SYNC.getConnectionId().toString()); @@ -353,19 +362,23 @@ public void testListJobs() throws IOException { @Test public void testListJobsWithStatus() throws IOException { // non-sync job that has failed. - final long discoverSchemaJobId = schedulerPersistence.createDiscoverSchemaJob(SOURCE_CONNECTION_IMPLEMENTATION); + final long discoverSchemaJobId = schedulerPersistence.createDiscoverSchemaJob(SOURCE_CONNECTION_IMPLEMENTATION, SOURCE_IMAGE_NAME); // sync job that is not failed. schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); // sync job that has failed. final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); schedulerPersistence.updateStatus(discoverSchemaJobId, JobStatus.FAILED); schedulerPersistence.updateStatus(jobId, JobStatus.FAILED); @@ -381,10 +394,20 @@ public void testListJobsWithStatus() throws IOException { @Test public void testGetLastSyncJobForConnectionId() throws IOException { - schedulerPersistence.createSyncJob(SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, STANDARD_SYNC); + schedulerPersistence.createSyncJob( + SOURCE_CONNECTION_IMPLEMENTATION, + DESTINATION_CONNECTION_IMPLEMENTATION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Instant afterNow = NOW.plusSeconds(1000); when(timeSupplier.get()).thenReturn(afterNow); - final long jobId = schedulerPersistence.createSyncJob(SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, STANDARD_SYNC); + final long jobId = schedulerPersistence.createSyncJob( + SOURCE_CONNECTION_IMPLEMENTATION, + DESTINATION_CONNECTION_IMPLEMENTATION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Optional actual = schedulerPersistence.getLastSyncJob(STANDARD_SYNC.getConnectionId()); @@ -432,13 +455,17 @@ public void testGetOldestPendingJob() throws IOException { final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Instant afterNow = NOW.plusSeconds(1000); when(timeSupplier.get()).thenReturn(afterNow); schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Optional actual = schedulerPersistence.getOldestPendingJob(); @@ -451,7 +478,9 @@ public void testGetOldestPendingJobOnlyPendingJobs() throws IOException, SQLExce final long jobId = schedulerPersistence.createSyncJob( SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, - STANDARD_SYNC); + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); DatabaseHelper.query( connectionPool, @@ -464,7 +493,12 @@ public void testGetOldestPendingJobOnlyPendingJobs() throws IOException, SQLExce @Test public void testGetJobFromRecord() throws IOException, SQLException { - final long jobId = schedulerPersistence.createSyncJob(SOURCE_CONNECTION_IMPLEMENTATION, DESTINATION_CONNECTION_IMPLEMENTATION, STANDARD_SYNC); + final long jobId = schedulerPersistence.createSyncJob( + SOURCE_CONNECTION_IMPLEMENTATION, + DESTINATION_CONNECTION_IMPLEMENTATION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + DESTINATION_IMAGE_NAME); final Record jobRecord = getJobRecord(jobId); final Job actual = DefaultSchedulerPersistence.getJobFromRecord(jobRecord); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 4a8c1bc8ef2a..12b9741529c1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -39,17 +39,22 @@ import io.airbyte.config.Schema; import io.airbyte.config.SourceConnectionImplementation; import io.airbyte.config.StandardCheckConnectionOutput; +import io.airbyte.config.StandardDestination; import io.airbyte.config.StandardDiscoverSchemaOutput; +import io.airbyte.config.StandardSource; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.persistence.SchedulerPersistence; +import io.airbyte.commons.docker.DockerUtil; import io.airbyte.server.converters.SchemaConverter; + import java.io.IOException; import java.util.Collections; import java.util.UUID; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,14 +77,16 @@ public CheckConnectionRead checkSourceImplementationConnection(SourceImplementat final SourceConnectionImplementation connectionImplementation = configRepository.getSourceConnectionImplementation(sourceImplementationIdRequestBody.getSourceImplementationId()); - final long jobId = schedulerPersistence.createSourceCheckConnectionJob(connectionImplementation); + final StandardSource source = configRepository.getStandardSource(connectionImplementation.getSourceId()); + final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final long jobId = schedulerPersistence.createSourceCheckConnectionJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); TrackingClientSingleton.get().track("check_connection", ImmutableMap.builder() .put("type", "source") .put("name", connectionImplementation.getName()) - .put("source_specification_id", connectionImplementation.getSourceSpecificationId()) + .put("source_id", connectionImplementation.getSourceId()) .put("source_implementation_id", connectionImplementation.getSourceImplementationId()) .put("check_connection_result", checkConnectionRead.getStatus()) .put("job_id", jobId) @@ -93,15 +100,17 @@ public CheckConnectionRead checkDestinationImplementationConnection(DestinationI final DestinationConnectionImplementation connectionImplementation = configRepository.getDestinationConnectionImplementation(destinationImplementationIdRequestBody.getDestinationImplementationId()); - final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(connectionImplementation); + final StandardDestination destination = configRepository.getStandardDestination(connectionImplementation.getDestinationId()); + final String imageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); TrackingClientSingleton.get().track("check_connection", ImmutableMap.builder() .put("type", "destination") .put("name", connectionImplementation.getName()) - .put("destination_specification_id", connectionImplementation.getDestinationSpecificationId()) .put("destination_implementation_id", connectionImplementation.getDestinationImplementationId()) + .put("destination_id", connectionImplementation.getDestinationId()) .put("check_connection_result", checkConnectionRead.getStatus()) .put("job_id", jobId) .build()); @@ -114,7 +123,9 @@ public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementat final SourceConnectionImplementation connectionImplementation = configRepository.getSourceConnectionImplementation(sourceImplementationIdRequestBody.getSourceImplementationId()); - final long jobId = schedulerPersistence.createDiscoverSchemaJob(connectionImplementation); + StandardSource source = configRepository.getStandardSource(connectionImplementation.getSourceId()); + final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final long jobId = schedulerPersistence.createDiscoverSchemaJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final Job job = waitUntilJobIsTerminalOrTimeout(jobId); @@ -126,7 +137,7 @@ public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementat TrackingClientSingleton.get().track("discover_schema", ImmutableMap.builder() .put("name", connectionImplementation.getName()) - .put("source_specification_id", connectionImplementation.getSourceSpecificationId()) + .put("source_id", connectionImplementation.getSourceId()) .put("source_implementation_id", connectionImplementation.getSourceImplementationId()) .put("job_id", jobId) .build()); @@ -144,17 +155,29 @@ public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectio final DestinationConnectionImplementation destinationConnectionImplementation = configRepository.getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()); - final long jobId = schedulerPersistence.createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync); + StandardSource source = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); + final String sourceImageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + + StandardSource destination = configRepository.getStandardSource(destinationConnectionImplementation.getDestinationId()); + final String destinationImageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + + final long jobId = schedulerPersistence.createSyncJob( + sourceConnectionImplementation, + destinationConnectionImplementation, + standardSync, + sourceImageName, + destinationImageName + ); final Job job = waitUntilJobIsTerminalOrTimeout(jobId); TrackingClientSingleton.get().track("sync", ImmutableMap.builder() .put("name", standardSync.getName()) .put("connection_id", standardSync.getConnectionId()) .put("sync_mode", standardSync.getSyncMode()) - .put("source_specification_id", sourceConnectionImplementation.getSourceSpecificationId()) + .put("source_id", sourceConnectionImplementation.getSourceId()) .put("source_implementation_id", sourceConnectionImplementation.getSourceImplementationId()) - .put("destination_specification_id", destinationConnectionImplementation.getDestinationSpecificationId()) .put("destination_implementation_id", destinationConnectionImplementation.getDestinationImplementationId()) + .put("destination_id", destinationConnectionImplementation.getDestinationId()) .put("job_id", jobId) .build()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java index 020b0a5c32c7..8975ce2b7815 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java @@ -176,10 +176,7 @@ private SourceImplementationRead buildSourceImplementationRead(UUID sourceImplem // read configuration from db final SourceConnectionImplementation sourceConnectionImplementation = configRepository.getSourceConnectionImplementation(sourceImplementationId); - final UUID sourceId = configRepository - .getSourceConnectionSpecification(sourceConnectionImplementation.getSourceSpecificationId()) - .getSourceId(); - final StandardSource standardSource = configRepository.getStandardSource(sourceId); + final StandardSource standardSource = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); return toSourceImplementationRead(sourceConnectionImplementation, standardSource); } @@ -199,7 +196,7 @@ private void persistSourceConnectionImplementation(final String name, throws JsonValidationException, IOException { final SourceConnectionImplementation sourceConnectionImplementation = new SourceConnectionImplementation() .withName(name) - .withSourceSpecificationId(sourceSpecificationId) + .withSourceId(sourceSpecificationId) .withWorkspaceId(workspaceId) .withSourceImplementationId(sourceImplementationId) .withTombstone(tombstone) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index aa4763e75ed0..292b2e2b893c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -56,6 +56,8 @@ class SchedulerHandlerTest { private static final long JOB_ID = 15L; + private static final String SOURCE_IMAGE_NAME = "srcimage"; + private static final String DESTINATION_IMAGE_NAME = "dstimage"; private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; private SchedulerPersistence schedulerPersistence; @@ -81,13 +83,13 @@ void testCheckSourceImplementationConnection() throws JsonValidationException, I new SourceImplementationIdRequestBody().sourceImplementationId(sourceImpl.getSourceImplementationId()); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); - when(schedulerPersistence.createSourceCheckConnectionJob(sourceImpl)).thenReturn(JOB_ID); + when(schedulerPersistence.createSourceCheckConnectionJob(sourceImpl, SOURCE_IMAGE_NAME)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.checkSourceImplementationConnection(request); verify(configRepository).getSourceConnectionImplementation(sourceImpl.getSourceImplementationId()); - verify(schedulerPersistence).createSourceCheckConnectionJob(sourceImpl); + verify(schedulerPersistence).createSourceCheckConnectionJob(sourceImpl, SOURCE_IMAGE_NAME); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @@ -98,13 +100,13 @@ void testCheckDestinationImplementationConnection() throws IOException, JsonVali new DestinationImplementationIdRequestBody().destinationImplementationId(destinationImpl.getDestinationImplementationId()); when(configRepository.getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId())).thenReturn(destinationImpl); - when(schedulerPersistence.createDestinationCheckConnectionJob(destinationImpl)).thenReturn(JOB_ID); + when(schedulerPersistence.createDestinationCheckConnectionJob(destinationImpl, DESTINATION_IMAGE_NAME)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.checkDestinationImplementationConnection(request); verify(configRepository).getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId()); - verify(schedulerPersistence).createDestinationCheckConnectionJob(destinationImpl); + verify(schedulerPersistence).createDestinationCheckConnectionJob(destinationImpl, DESTINATION_IMAGE_NAME); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @@ -115,13 +117,13 @@ void testDiscoverSchemaForSourceImplementation() throws IOException, JsonValidat new SourceImplementationIdRequestBody().sourceImplementationId(sourceImpl.getSourceImplementationId()); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); - when(schedulerPersistence.createDiscoverSchemaJob(sourceImpl)).thenReturn(JOB_ID); + when(schedulerPersistence.createDiscoverSchemaJob(sourceImpl, SOURCE_IMAGE_NAME)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.discoverSchemaForSourceImplementation(request); verify(configRepository).getSourceConnectionImplementation(sourceImpl.getSourceImplementationId()); - verify(schedulerPersistence).createDiscoverSchemaJob(sourceImpl); + verify(schedulerPersistence).createDiscoverSchemaJob(sourceImpl, SOURCE_IMAGE_NAME); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @@ -137,7 +139,7 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); when(configRepository.getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId())).thenReturn(destinationImpl); - when(schedulerPersistence.createSyncJob(sourceImpl, destinationImpl, standardSync)).thenReturn(JOB_ID); + when(schedulerPersistence.createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_IMAGE_NAME, DESTINATION_IMAGE_NAME)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.syncConnection(request); @@ -145,7 +147,7 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot verify(configRepository).getStandardSync(standardSync.getConnectionId()); verify(configRepository).getSourceConnectionImplementation(standardSync.getSourceImplementationId()); verify(configRepository).getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()); - verify(schedulerPersistence).createSyncJob(sourceImpl, destinationImpl, standardSync); + verify(schedulerPersistence).createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_IMAGE_NAME, DESTINATION_IMAGE_NAME); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java b/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java index f3f454c4b8ed..944d498e05ef 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/TestConfigHelpers.java @@ -49,9 +49,9 @@ public class TestConfigHelpers { public static ImmutablePair createSyncConfig() { final UUID workspaceId = UUID.randomUUID(); - final UUID sourceSpecificationId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); final UUID sourceImplementationId = UUID.randomUUID(); - final UUID destinationSpecificationId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); final UUID destinationImplementationId = UUID.randomUUID(); final UUID connectionId = UUID.randomUUID(); @@ -70,14 +70,14 @@ public static ImmutablePair createSyncConfig() final SourceConnectionImplementation sourceConnectionConfig = new SourceConnectionImplementation() .withConfiguration(sourceConnection) .withWorkspaceId(workspaceId) - .withSourceSpecificationId(sourceSpecificationId) + .withSourceId(sourceId) .withSourceImplementationId(sourceImplementationId) .withTombstone(false); final DestinationConnectionImplementation destinationConnectionConfig = new DestinationConnectionImplementation() .withConfiguration(destinationConnection) .withWorkspaceId(workspaceId) - .withDestinationSpecificationId(destinationSpecificationId) + .withDestinationId(destinationId) .withDestinationImplementationId(destinationImplementationId) .withTombstone(false); From 1e0aa2c4286c6ff5508f0b95756986100ecbf388 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Thu, 8 Oct 2020 21:58:08 -0700 Subject: [PATCH 03/12] save --- airbyte-api/src/main/openapi/config.yaml | 34 +++++++++---------- .../config/persistence/ConfigRepository.java | 16 --------- .../SourceImplementationsHandler.java | 25 ++++++++------ 3 files changed, 32 insertions(+), 43 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index fdcb41e45a8a..7339692e2325 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -944,41 +944,41 @@ components: type: object required: - name - - defaultDockerRepository - - defaultDockerImageVersion + - dockerRepository + - dockerImageTag properties: name: type: string - defaultDockerRepository: + dockerRepository: type: string - defaultDockerImageVersion: + dockerImageTag: type: string SourceUpdate: type: object description: Update the source. Currently, the only allowed attribute to update is the default docker image version. required: - sourceId - - defaultDockerImageVersion + - dockerImageTag properties: sourceId: $ref: "#/components/schemas/SourceId" - defaultDockerImageVersion: + dockerImageTag: type: string SourceRead: type: object required: - sourceId - name - - defaultDockerRepository - - defaultDockerImageVersion + - dockerRepository + - dockerImageVersion properties: sourceId: $ref: "#/components/schemas/SourceId" name: type: string - defaultDockerRepository: + dockerRepository: type: string - defaultDockerImageVersion: + dockerImageVersion: type: string SourceReadList: type: object @@ -997,7 +997,7 @@ components: type: object required: - sourceId - - specification + - connectionSpecification properties: sourceId: $ref: "#/components/schemas/SourceId" @@ -1105,23 +1105,23 @@ components: properties: destinationId: $ref: "#/components/schemas/DestinationId" - defaultDockerImageVersion: + dockerImageTag: type: string DestinationRead: type: object required: - destinationId - name - - defaultDockerRepository - - defaultDockerImageVersion + - dockerRepository + - dockerImageTag properties: destinationId: $ref: "#/components/schemas/DestinationId" name: type: string - defaultDockerRepository: + dockerRepository: type: string - defaultDockerImageVersion: + dockerImageTag: type: string DestinationReadList: type: object @@ -1140,7 +1140,7 @@ components: type: object required: - destinationId - - specification + - connectionSpecification properties: destinationId: $ref: "#/components/schemas/DestinationId" diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 9e1cde0a5fb8..fbcceda316b6 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -91,14 +91,6 @@ public List listStandardDestinations() return persistence.listConfigs(ConfigSchema.STANDARD_DESTINATION, StandardDestination.class); } - public SourceConnectionSpecification getSourceConnectionSpecification(final UUID sourceSpecificationId) - throws JsonValidationException, IOException, ConfigNotFoundException { - return persistence.getConfig( - ConfigSchema.SOURCE_CONNECTION_SPECIFICATION, - sourceSpecificationId.toString(), - SourceConnectionSpecification.class); - } - public List listSourceConnectionSpecifications() throws JsonValidationException, IOException, ConfigNotFoundException { return persistence.listConfigs( @@ -106,14 +98,6 @@ public List listSourceConnectionSpecifications() SourceConnectionSpecification.class); } - public DestinationConnectionSpecification getDestinationConnectionSpecification(final UUID destinationSpecificationId) - throws JsonValidationException, IOException, ConfigNotFoundException { - return persistence.getConfig( - ConfigSchema.DESTINATION_CONNECTION_SPECIFICATION, - destinationSpecificationId.toString(), - DestinationConnectionSpecification.class); - } - public List listDestinationConnectionSpecifications() throws JsonValidationException, IOException, ConfigNotFoundException { return persistence.listConfigs( diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java index 8975ce2b7815..2e1f7b822db2 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java @@ -42,6 +42,7 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.server.validation.IntegrationSchemaValidation; + import java.io.IOException; import java.util.List; import java.util.UUID; @@ -52,36 +53,40 @@ public class SourceImplementationsHandler { private final Supplier uuidGenerator; private final ConfigRepository configRepository; private final IntegrationSchemaValidation validator; + private SchedulerHandler schedulerHandler; private final ConnectionsHandler connectionsHandler; public SourceImplementationsHandler(final ConfigRepository configRepository, final IntegrationSchemaValidation integrationSchemaValidation, + final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler, final Supplier uuidGenerator) { this.configRepository = configRepository; this.validator = integrationSchemaValidation; + this.schedulerHandler = schedulerHandler; this.connectionsHandler = connectionsHandler; this.uuidGenerator = uuidGenerator; } public SourceImplementationsHandler(final ConfigRepository configRepository, final IntegrationSchemaValidation integrationSchemaValidation, + final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler) { - this(configRepository, integrationSchemaValidation, connectionsHandler, UUID::randomUUID); + this(configRepository, integrationSchemaValidation, schedulerHandler, connectionsHandler, UUID::randomUUID); } public SourceImplementationRead createSourceImplementation(SourceImplementationCreate sourceImplementationCreate) throws ConfigNotFoundException, IOException, JsonValidationException { // validate configuration validateSourceImplementation( - sourceImplementationCreate.getSourceSpecificationId(), + sourceImplementationCreate.getSourceId(), sourceImplementationCreate.getConnectionConfiguration()); // persist final UUID sourceImplementationId = uuidGenerator.get(); persistSourceConnectionImplementation( sourceImplementationCreate.getName() != null ? sourceImplementationCreate.getName() : "default", - sourceImplementationCreate.getSourceSpecificationId(), + sourceImplementationCreate.getSourceId(), sourceImplementationCreate.getWorkspaceId(), sourceImplementationId, false, @@ -99,13 +104,13 @@ public SourceImplementationRead updateSourceImplementation(SourceImplementationU // validate configuration validateSourceImplementation( - persistedSourceImplementation.getSourceSpecificationId(), + persistedSourceImplementation.getSourceId(), sourceImplementationUpdate.getConnectionConfiguration()); // persist persistSourceConnectionImplementation( sourceImplementationUpdate.getName(), - persistedSourceImplementation.getSourceSpecificationId(), + persistedSourceImplementation.getSourceId(), persistedSourceImplementation.getWorkspaceId(), sourceImplementationUpdate.getSourceImplementationId(), persistedSourceImplementation.getTombstone(), @@ -164,7 +169,7 @@ public void deleteSourceImplementation(SourceImplementationIdRequestBody sourceI // persist persistSourceConnectionImplementation( sourceImplementation.getName(), - sourceImplementation.getSourceSpecificationId(), + sourceImplementation.getSourceId(), sourceImplementation.getWorkspaceId(), sourceImplementation.getSourceImplementationId(), true, @@ -183,12 +188,12 @@ private SourceImplementationRead buildSourceImplementationRead(UUID sourceImplem private void validateSourceImplementation(UUID sourceConnectionSpecificationId, JsonNode implementationJson) throws JsonValidationException, IOException, ConfigNotFoundException { - SourceConnectionSpecification scs = configRepository.getSourceConnectionSpecification(sourceConnectionSpecificationId); + SourceConnectionSpecification scs = schedulerHandler.getSourceConnectionSpecification(sourceConnectionSpecificationId); validator.validateConfig(scs, implementationJson); } private void persistSourceConnectionImplementation(final String name, - final UUID sourceSpecificationId, + final UUID sourceId, final UUID workspaceId, final UUID sourceImplementationId, final boolean tombstone, @@ -196,7 +201,7 @@ private void persistSourceConnectionImplementation(final String name, throws JsonValidationException, IOException { final SourceConnectionImplementation sourceConnectionImplementation = new SourceConnectionImplementation() .withName(name) - .withSourceId(sourceSpecificationId) + .withSourceId(sourceId) .withWorkspaceId(workspaceId) .withSourceImplementationId(sourceImplementationId) .withTombstone(tombstone) @@ -212,7 +217,7 @@ private SourceImplementationRead toSourceImplementationRead(final SourceConnecti .sourceName(standardSource.getName()) .sourceImplementationId(sourceConnectionImplementation.getSourceImplementationId()) .workspaceId(sourceConnectionImplementation.getWorkspaceId()) - .sourceSpecificationId(sourceConnectionImplementation.getSourceSpecificationId()) + .sourceId(sourceConnectionImplementation.getSourceId()) .connectionConfiguration(sourceConnectionImplementation.getConfiguration()) .name(sourceConnectionImplementation.getName()); } From 578f0837d3ac2a15bfe8b71576b9b52beb47e443 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 01:14:23 -0700 Subject: [PATCH 04/12] successful tests --- airbyte-api/src/main/openapi/config.yaml | 134 +++++++++--------- .../io/airbyte/commons/docker/DockerUtil.java | 28 +++- .../commons/docker/DockerUtilTest.java | 29 +++- .../java/io/airbyte/config/ConfigSchema.java | 5 +- .../DestinationConnectionSpecification.yaml | 24 ---- .../types/SourceConnectionSpecification.yaml | 26 ---- .../io/airbyte/config/ConfigSchemaTest.java | 2 +- .../config/persistence/ConfigRepository.java | 5 +- .../persistence/ConfigRepositoryTest.java | 80 ----------- .../integrations/base/Destination.java | 4 +- .../base/IntegrationRunnerTest.java | 5 +- .../destination/csv/CsvDestination.java | 6 +- .../src/main/resources/spec.json | 4 +- .../destination/csv/CsvDestinationTest.java | 6 +- .../template/DestinationTemplate.java | 6 +- .../job_factory/DefaultSyncJobFactory.java | 1 - .../DefaultSchedulerPersistence.java | 4 +- .../DefaultSyncJobFactoryTest.java | 29 +++- .../DefaultSchedulerPersistenceTest.java | 24 +--- .../airbyte/server/apis/ConfigurationApi.java | 9 +- .../DestinationImplementationsHandler.java | 45 +++--- .../server/handlers/SchedulerHandler.java | 36 ++--- .../SourceImplementationsHandler.java | 19 +-- .../IntegrationSchemaValidation.java | 57 -------- ...DestinationImplementationsHandlerTest.java | 82 +++++------ .../server/handlers/SchedulerHandlerTest.java | 101 ++++++++----- .../SourceImplementationsHandlerTest.java | 61 ++++---- ...dDestinationImplementationHandlerTest.java | 4 +- ...ackendSourceImplementationHandlerTest.java | 3 +- ...ava => ConnectorSpecificationHelpers.java} | 26 ++-- .../DestinationImplementationHelpers.java | 6 +- .../DestinationSpecificationHelpers.java | 55 ------- .../helpers/SourceImplementationHelpers.java | 6 +- 33 files changed, 386 insertions(+), 546 deletions(-) delete mode 100644 airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml delete mode 100644 airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml delete mode 100644 airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryTest.java delete mode 100644 airbyte-server/src/main/java/io/airbyte/server/validation/IntegrationSchemaValidation.java rename airbyte-server/src/test/java/io/airbyte/server/helpers/{SourceSpecificationHelpers.java => ConnectorSpecificationHelpers.java} (66%) delete mode 100644 airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationSpecificationHelpers.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 7339692e2325..80abc8747a05 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -121,48 +121,48 @@ paths: description: Workspace not found "422": $ref: "#/components/responses/InvalidInput" - /v1/sources/create: - post: - tags: - - source - summary: Creates a source - operationId: createSource - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SourceCreate" - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SourceRead" - "422": - $ref: "#/components/responses/InvalidInput" - /v1/sources/update: - post: - tags: - - source - summary: Update a source - operationId: updateSource - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SourceUpdate" - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/SourceRead" - "404": - description: Source not found - "422": - $ref: "#/components/responses/InvalidInput" + # /v1/sources/create: + # post: + # tags: + # - source + # summary: Creates a source + # operationId: createSource + # requestBody: + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/SourceCreate" + # responses: + # "200": + # description: Successful operation + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/SourceRead" + # "422": + # $ref: "#/components/responses/InvalidInput" + # /v1/sources/update: + # post: + # tags: + # - source + # summary: Update a source + # operationId: updateSource + # requestBody: + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/SourceUpdate" + # responses: + # "200": + # description: Successful operation + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/SourceRead" + # "404": + # description: Source not found + # "422": + # $ref: "#/components/responses/InvalidInput" /v1/sources/list: post: tags: @@ -379,29 +379,29 @@ paths: description: Source Implementation not found "422": $ref: "#/components/responses/InvalidInput" - /v1/destinations/update: - post: - tags: - - destination - summary: Update destination - operationId: updateDestination - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/DestinationUpdate" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/DestinationRead" - "404": - description: Destination not found - "422": - $ref: "#/components/responses/InvalidInput" + # /v1/destinations/update: + # post: + # tags: + # - destination + # summary: Update destination + # operationId: updateDestination + # requestBody: + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/DestinationUpdate" + # required: true + # responses: + # "200": + # description: Successful operation + # content: + # application/json: + # schema: + # $ref: "#/components/schemas/DestinationRead" + # "404": + # description: Destination not found + # "422": + # $ref: "#/components/responses/InvalidInput" /v1/destinations/list: post: tags: @@ -1401,6 +1401,7 @@ components: - check_connection_source - check_connection_destination - discover_schema + - get_spec - sync JobListRequestBody: type: object @@ -1512,6 +1513,7 @@ components: - syncSchema - status - source + - isSyncing properties: connectionId: $ref: "#/components/schemas/ConnectionId" @@ -1538,6 +1540,8 @@ components: description: epoch time of last sync. null if no sync has taken place. type: integer format: int64 + isSyncing: + type: boolean WbConnectionReadList: type: object required: diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java index 5b6e339b5c79..c5b22e6be249 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java @@ -1,7 +1,33 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + package io.airbyte.commons.docker; public class DockerUtil { - public static String getTaggedImageName(String dockerRepository, String tag){ + + public static String getTaggedImageName(String dockerRepository, String tag) { return String.join(":", dockerRepository, tag); } + } diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java index f0c3ade13193..703ee6defaad 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java @@ -1,9 +1,33 @@ -package io.airbyte.commons.docker; +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ -import org.junit.jupiter.api.Test; +package io.airbyte.commons.docker; import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + class DockerUtilTest { @Test @@ -12,4 +36,5 @@ void testGetTaggedImageName() { String tag = "12.3"; assertEquals("airbyte/repo:12.3", DockerUtil.getTaggedImageName(repository, tag)); } + } diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java index 5e520f1b570e..2d9ec26f5638 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/ConfigSchema.java @@ -40,14 +40,15 @@ public enum ConfigSchema { // source STANDARD_SOURCE("StandardSource.yaml"), - SOURCE_CONNECTION_SPECIFICATION("SourceConnectionSpecification.yaml"), SOURCE_CONNECTION_IMPLEMENTATION("SourceConnectionImplementation.yaml"), // destination STANDARD_DESTINATION("StandardDestination.yaml"), - DESTINATION_CONNECTION_SPECIFICATION("DestinationConnectionSpecification.yaml"), DESTINATION_CONNECTION_IMPLEMENTATION("DestinationConnectionImplementation.yaml"), + // specs + CONNECTOR_SPECIFICATION("ConnectorSpecification.yaml"), + // sync STANDARD_SYNC("StandardSync.yaml"), STANDARD_SYNC_SUMMARY("StandardSyncSummary.yaml"), diff --git a/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml b/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml deleted file mode 100644 index dd6945f54c15..000000000000 --- a/airbyte-config/models/src/main/resources/types/DestinationConnectionSpecification.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -"$schema": http://json-schema.org/draft-07/schema# -"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/DestinationSpecification.yaml -title: DestinationConnectionSpecification -description: specification for how to configure a connection to a destination -type: object -required: - - destinationId - - destinationSpecificationId - - specification -additionalProperties: false -properties: - destinationId: - type: string - format: uuid - destinationSpecificationId: - type: string - format: uuid - documentationUrl: - type: string - specification: - description: Integration specific blob. Must be a valid JSON string. - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml b/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml deleted file mode 100644 index 65a5455921ef..000000000000 --- a/airbyte-config/models/src/main/resources/types/SourceConnectionSpecification.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -"$schema": http://json-schema.org/draft-07/schema# -"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/SourceSpecification.yaml -title: SourceConnectionSpecification -description: specification for how to configure a connection to a source -type: object -required: - - sourceId - - sourceSpecificationId - - specification -additionalProperties: false -properties: - sourceId: - type: string - format: uuid - sourceSpecificationId: - type: string - format: uuid - documentationUrl: - type: string - changelogUrl: - type: string - specification: - description: Integration specific blob. Must be a valid JSON string. - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/ConfigSchemaTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/ConfigSchemaTest.java index 0fdf34fb907c..f77ab59b66a7 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/ConfigSchemaTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/ConfigSchemaTest.java @@ -42,7 +42,7 @@ void testFile() throws IOException { @Test void testPrepareKnownSchemas() { for (ConfigSchema value : ConfigSchema.values()) { - assertTrue(Files.exists(value.getFile().toPath())); + assertTrue(Files.exists(value.getFile().toPath()), value.getFile().toPath().toString() + " does not exist"); } } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 251ad72789a8..fb3c97940f13 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -33,12 +33,11 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSyncSchedule; import io.airbyte.config.StandardWorkspace; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.List; import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ConfigRepository { diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryTest.java deleted file mode 100644 index 13cf1df20f82..000000000000 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.config.persistence; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.common.collect.Lists; -import io.airbyte.commons.json.JsonValidationException; -import io.airbyte.config.ConfigSchema; -import io.airbyte.config.DestinationConnectionSpecification; -import io.airbyte.config.SourceConnectionSpecification; -import java.io.IOException; -import java.util.ArrayList; -import java.util.UUID; -import org.junit.jupiter.api.Test; - -class ConfigRepositoryTest { - - private ConfigPersistence configPersistence = mock(ConfigPersistence.class); - private ConfigRepository repository = new ConfigRepository(configPersistence); - - @Test - void getSourceConnectionSpecificationFromSourceId() throws JsonValidationException, IOException, ConfigNotFoundException { - UUID id1 = UUID.randomUUID(); - UUID id2 = UUID.randomUUID(); - - SourceConnectionSpecification spec1 = new SourceConnectionSpecification().withSourceId(id1); - SourceConnectionSpecification spec2 = new SourceConnectionSpecification().withSourceId(id2); - - ArrayList specs = Lists.newArrayList(spec1, spec2); - when(configPersistence.listConfigs(ConfigSchema.SOURCE_CONNECTION_SPECIFICATION, SourceConnectionSpecification.class)) - .thenReturn(specs); - - SourceConnectionSpecification actual = repository.getSourceConnectionSpecificationFromSourceId(id1); - - assertEquals(spec1, actual); - } - - @Test - void getDestinationConnectionSpecificationFromDestinationId() throws JsonValidationException, IOException, ConfigNotFoundException { - UUID id1 = UUID.randomUUID(); - UUID id2 = UUID.randomUUID(); - - DestinationConnectionSpecification spec1 = new DestinationConnectionSpecification().withDestinationId(id1); - DestinationConnectionSpecification spec2 = new DestinationConnectionSpecification().withDestinationId(id2); - - ArrayList specs = Lists.newArrayList(spec1, spec2); - when(configPersistence.listConfigs(ConfigSchema.DESTINATION_CONNECTION_SPECIFICATION, DestinationConnectionSpecification.class)) - .thenReturn(specs); - - DestinationConnectionSpecification actual = repository.getDestinationConnectionSpecificationFromDestinationId(id1); - - assertEquals(spec1, actual); - } - -} diff --git a/airbyte-integrations/base-java/src/main/java/io/airbyte/integrations/base/Destination.java b/airbyte-integrations/base-java/src/main/java/io/airbyte/integrations/base/Destination.java index fa5f344ee4d8..159c45b3b3e7 100644 --- a/airbyte-integrations/base-java/src/main/java/io/airbyte/integrations/base/Destination.java +++ b/airbyte-integrations/base-java/src/main/java/io/airbyte/integrations/base/Destination.java @@ -25,7 +25,7 @@ package io.airbyte.integrations.base; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.config.DestinationConnectionSpecification; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.Schema; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDiscoverSchemaOutput; @@ -40,7 +40,7 @@ public interface Destination { * @return specification. * @throws Exception - any exception. */ - DestinationConnectionSpecification spec() throws Exception; + ConnectorSpecification spec() throws Exception; /** * Check whether, given the current configuration, the integration can connect to the destination. diff --git a/airbyte-integrations/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java b/airbyte-integrations/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java index 6a16bfc17912..8e03896315c0 100644 --- a/airbyte-integrations/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java +++ b/airbyte-integrations/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java @@ -37,7 +37,7 @@ import com.google.common.collect.Lists; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationConnectionSpecification; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.Schema; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardCheckConnectionOutput.Status; @@ -46,6 +46,7 @@ import io.airbyte.singer.SingerMessage; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.function.Consumer; @@ -87,7 +88,7 @@ void setup() throws IOException { @Test void testSpec() throws Exception { final IntegrationConfig intConfig = IntegrationConfig.spec(); - final DestinationConnectionSpecification output = new DestinationConnectionSpecification().withDocumentationUrl("https://docs.airbyte.io/"); + final ConnectorSpecification output = new ConnectorSpecification().withDocumentationUrl(new URI("https://docs.airbyte.io/")); when(cliParser.parse(ARGS)).thenReturn(intConfig); when(destination.spec()).thenReturn(output); diff --git a/airbyte-integrations/csv-destination/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java b/airbyte-integrations/csv-destination/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java index 70b8c6ac6c26..ca1b1dbdfe37 100644 --- a/airbyte-integrations/csv-destination/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java +++ b/airbyte-integrations/csv-destination/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java @@ -28,7 +28,7 @@ import com.google.common.base.Preconditions; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.config.DestinationConnectionSpecification; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.Schema; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardCheckConnectionOutput.Status; @@ -62,9 +62,9 @@ public class CsvDestination implements Destination { static final String DESTINATION_PATH_FIELD = "destination_path"; @Override - public DestinationConnectionSpecification spec() throws IOException { + public ConnectorSpecification spec() throws IOException { final String resourceString = MoreResources.readResource("spec.json"); - return Jsons.deserialize(resourceString, DestinationConnectionSpecification.class); + return Jsons.deserialize(resourceString, ConnectorSpecification.class); } @Override diff --git a/airbyte-integrations/csv-destination/src/main/resources/spec.json b/airbyte-integrations/csv-destination/src/main/resources/spec.json index 99c54a269b0c..4dba528498b0 100644 --- a/airbyte-integrations/csv-destination/src/main/resources/spec.json +++ b/airbyte-integrations/csv-destination/src/main/resources/spec.json @@ -1,8 +1,6 @@ { - "destinationId": "", - "destinationSpecificationId": "", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-csv", - "specification": { + "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "CSV Destination Spec", "type": "object", diff --git a/airbyte-integrations/csv-destination/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java b/airbyte-integrations/csv-destination/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java index 40c72b293c55..4771565f7ca0 100644 --- a/airbyte-integrations/csv-destination/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java +++ b/airbyte-integrations/csv-destination/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java @@ -35,8 +35,8 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.DataType; -import io.airbyte.config.DestinationConnectionSpecification; import io.airbyte.config.Field; import io.airbyte.config.Schema; import io.airbyte.config.StandardCheckConnectionOutput; @@ -98,9 +98,9 @@ void setup() throws IOException { @Test void testSpec() throws IOException { - final DestinationConnectionSpecification actual = new CsvDestination().spec(); + final ConnectorSpecification actual = new CsvDestination().spec(); final String resourceString = MoreResources.readResource("spec.json"); - final DestinationConnectionSpecification expected = Jsons.deserialize(resourceString, DestinationConnectionSpecification.class); + final ConnectorSpecification expected = Jsons.deserialize(resourceString, ConnectorSpecification.class); assertEquals(expected, actual); } diff --git a/airbyte-integrations/java-template-destination/src/main/java/io/airbyte/integrations/destination/template/DestinationTemplate.java b/airbyte-integrations/java-template-destination/src/main/java/io/airbyte/integrations/destination/template/DestinationTemplate.java index 3a8badef0845..a63ded3f5a79 100644 --- a/airbyte-integrations/java-template-destination/src/main/java/io/airbyte/integrations/destination/template/DestinationTemplate.java +++ b/airbyte-integrations/java-template-destination/src/main/java/io/airbyte/integrations/destination/template/DestinationTemplate.java @@ -27,7 +27,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.config.DestinationConnectionSpecification; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.Schema; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDiscoverSchemaOutput; @@ -48,10 +48,10 @@ public class DestinationTemplate implements Destination { // the code in this method uses this suggestion. replace it if you'd like to take a different // approach. @Override - public DestinationConnectionSpecification spec() throws IOException { + public ConnectorSpecification spec() throws IOException { // return a jsonschema representation of the spec for the integration. final String resourceString = MoreResources.readResource("spec.json"); - return Jsons.deserialize(resourceString, DestinationConnectionSpecification.class); + return Jsons.deserialize(resourceString, ConnectorSpecification.class); } // fixme - implement this method such that it checks whether it can connect to the destination. diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java index e18528986219..39ce920cf9f1 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java @@ -34,7 +34,6 @@ import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.scheduler.persistence.SchedulerPersistence; - import java.io.IOException; import java.util.UUID; diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java index 4429596fa307..2e7856288cc7 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistence.java @@ -37,7 +37,6 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardSyncOutput; import io.airbyte.db.DatabaseHelper; -import io.airbyte.integrations.Integrations; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.ScopeHelper; @@ -92,7 +91,8 @@ public long createSourceCheckConnectionJob(SourceConnectionImplementation source } @Override - public long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation, String dockerImageName) throws IOException { + public long createDestinationCheckConnectionJob(DestinationConnectionImplementation destinationImplementation, String dockerImageName) + throws IOException { final String scope = ScopeHelper.createScope( JobConfig.ConfigType.CHECK_CONNECTION_DESTINATION, diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java index 1af1f3f41405..3fcb0ce2f1d9 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java @@ -29,9 +29,12 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.airbyte.commons.docker.DockerUtil; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.DestinationConnectionImplementation; import io.airbyte.config.SourceConnectionImplementation; +import io.airbyte.config.StandardDestination; +import io.airbyte.config.StandardSource; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -44,6 +47,8 @@ class DefaultSyncJobFactoryTest { @Test void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID sourceId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); final UUID connectionId = UUID.randomUUID(); final UUID sourceImplId = UUID.randomUUID(); final UUID destinationImplId = UUID.randomUUID(); @@ -55,19 +60,35 @@ void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFo .withSourceImplementationId(sourceImplId) .withDestinationImplementationId(destinationImplId); - final SourceConnectionImplementation sourceConnectionImplementation = new SourceConnectionImplementation(); - final DestinationConnectionImplementation destinationConnectionImplementation = new DestinationConnectionImplementation(); + final SourceConnectionImplementation sourceConnectionImplementation = new SourceConnectionImplementation().withSourceId(sourceId); + final DestinationConnectionImplementation destinationConnectionImplementation = + new DestinationConnectionImplementation().withDestinationId(destinationId); + final String srcDockerRepo = "srcrepo"; + final String srcDockerTag = "tag"; + final String srcDockerImage = DockerUtil.getTaggedImageName(srcDockerRepo, srcDockerTag); + + final String dstDockerRepo = "dstrepo"; + final String dstDockerTag = "tag"; + final String dstDockerImage = DockerUtil.getTaggedImageName(dstDockerRepo, dstDockerTag); when(configRepository.getStandardSync(connectionId)).thenReturn(standardSync); when(configRepository.getSourceConnectionImplementation(sourceImplId)).thenReturn(sourceConnectionImplementation); when(configRepository.getDestinationConnectionImplementation(destinationImplId)).thenReturn(destinationConnectionImplementation); - when(schedulerPersistence.createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync)).thenReturn(jobId); + when(schedulerPersistence + .createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync, srcDockerImage, dstDockerImage)) + .thenReturn(jobId); + when(configRepository.getStandardSource(sourceId)) + .thenReturn(new StandardSource().withSourceId(sourceId).withDockerRepository(srcDockerRepo).withDockerImageTag(srcDockerTag)); + + when(configRepository.getStandardDestination(destinationId)) + .thenReturn(new StandardDestination().withDestinationId(destinationId).withDockerRepository(dstDockerRepo).withDockerImageTag(dstDockerTag)); final SyncJobFactory factory = new DefaultSyncJobFactory(schedulerPersistence, configRepository); final long actualJobId = factory.create(connectionId); assertEquals(jobId, actualJobId); - verify(schedulerPersistence).createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync); + verify(schedulerPersistence) + .createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync, srcDockerImage, dstDockerImage); } } diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java index 880c5454163f..8bf52af8e54b 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java @@ -53,7 +53,6 @@ import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.ScopeHelper; - import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; @@ -64,7 +63,6 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; - import org.apache.commons.dbcp2.BasicDataSource; import org.jooq.Record; import org.junit.jupiter.api.AfterAll; @@ -94,7 +92,7 @@ class DefaultSchedulerPersistenceTest { static { final UUID workspaceId = UUID.randomUUID(); final UUID sourceImplementationId = UUID.randomUUID(); - final UUID sourceSpecificationId = Integrations.POSTGRES_TAP.getSpecId(); + final UUID sourceId = UUID.randomUUID(); JsonNode implementationJson = Jsons.jsonNode(ImmutableMap.builder() .put("apiKey", "123-abc") @@ -103,7 +101,7 @@ class DefaultSchedulerPersistenceTest { SOURCE_CONNECTION_IMPLEMENTATION = new SourceConnectionImplementation() .withWorkspaceId(workspaceId) - .withSourceSpecificationId(sourceSpecificationId) + .withSourceId(sourceId) .withSourceImplementationId(sourceImplementationId) .withConfiguration(implementationJson) .withTombstone(false); @@ -113,7 +111,7 @@ class DefaultSchedulerPersistenceTest { DESTINATION_CONNECTION_IMPLEMENTATION = new DestinationConnectionImplementation() .withWorkspaceId(workspaceId) - .withDestinationSpecificationId(destinationSpecificationId) + .withDestinationId(destinationSpecificationId) .withDestinationImplementationId(destinationImplementationId) .withConfiguration(implementationJson) .withTombstone(false); @@ -426,14 +424,11 @@ public void testGetLastSyncJobForConnectionId() throws IOException { final Optional actual = schedulerPersistence.getLastSyncJob(STANDARD_SYNC.getConnectionId()); - final String sourceImageName = Integrations.findBySpecId(SOURCE_CONNECTION_IMPLEMENTATION.getSourceSpecificationId()).getTaggedImage(); - final String destinationImageName = - Integrations.findBySpecId(DESTINATION_CONNECTION_IMPLEMENTATION.getDestinationSpecificationId()).getTaggedImage(); final JobSyncConfig jobSyncConfig = new JobSyncConfig() .withSourceConnectionImplementation(SOURCE_CONNECTION_IMPLEMENTATION) - .withSourceDockerImage(sourceImageName) + .withSourceDockerImage(SOURCE_IMAGE_NAME) .withDestinationConnectionImplementation(DESTINATION_CONNECTION_IMPLEMENTATION) - .withDestinationDockerImage(destinationImageName) + .withDestinationDockerImage(DESTINATION_IMAGE_NAME) .withStandardSync(STANDARD_SYNC); final JobConfig jobConfig = new JobConfig() @@ -539,14 +534,9 @@ private Job getExpectedJob(long jobId) { } private Job getExpectedJob(long jobId, JobStatus jobStatus) { - final String sourceImageName = Integrations.findBySpecId(SOURCE_CONNECTION_IMPLEMENTATION.getSourceSpecificationId()) - .getTaggedImage(); - final String destinationImageName = Integrations.findBySpecId(DESTINATION_CONNECTION_IMPLEMENTATION.getDestinationSpecificationId()) - .getTaggedImage(); - final JobSyncConfig jobSyncConfig = new JobSyncConfig() - .withSourceDockerImage(sourceImageName) - .withDestinationDockerImage(destinationImageName) + .withSourceDockerImage(SOURCE_IMAGE_NAME) + .withDestinationDockerImage(DESTINATION_IMAGE_NAME) .withSourceConnectionImplementation(SOURCE_CONNECTION_IMPLEMENTATION) .withDestinationConnectionImplementation(DESTINATION_CONNECTION_IMPLEMENTATION) .withState(null) diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 7c5316b27aa8..533e708059d3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -61,6 +61,7 @@ import io.airbyte.api.model.WorkspaceIdRequestBody; import io.airbyte.api.model.WorkspaceRead; import io.airbyte.api.model.WorkspaceUpdate; +import io.airbyte.commons.json.JsonSchemaValidator; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -78,7 +79,6 @@ import io.airbyte.server.handlers.WebBackendDestinationImplementationHandler; import io.airbyte.server.handlers.WebBackendSourceImplementationHandler; import io.airbyte.server.handlers.WorkspacesHandler; -import io.airbyte.server.validation.IntegrationSchemaValidation; import java.io.IOException; import javax.validation.Valid; import org.eclipse.jetty.http.HttpStatus; @@ -100,14 +100,15 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { private final WebBackendDestinationImplementationHandler webBackendDestinationImplementationHandler; public ConfigurationApi(final ConfigRepository configRepository, final SchedulerPersistence schedulerPersistence) { - final IntegrationSchemaValidation integrationSchemaValidation = new IntegrationSchemaValidation(); + final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); workspacesHandler = new WorkspacesHandler(configRepository); sourcesHandler = new SourcesHandler(configRepository); connectionsHandler = new ConnectionsHandler(configRepository); - sourceImplementationsHandler = new SourceImplementationsHandler(configRepository, integrationSchemaValidation, connectionsHandler); destinationsHandler = new DestinationsHandler(configRepository); - destinationImplementationsHandler = new DestinationImplementationsHandler(configRepository, integrationSchemaValidation, connectionsHandler); schedulerHandler = new SchedulerHandler(configRepository, schedulerPersistence); + destinationImplementationsHandler = + new DestinationImplementationsHandler(configRepository, schemaValidator, schedulerHandler, connectionsHandler); + sourceImplementationsHandler = new SourceImplementationsHandler(configRepository, schemaValidator, schedulerHandler, connectionsHandler); jobHistoryHandler = new JobHistoryHandler(schedulerPersistence); webBackendConnectionsHandler = new WebBackendConnectionsHandler(connectionsHandler, sourceImplementationsHandler, jobHistoryHandler); webBackendSourceImplementationHandler = new WebBackendSourceImplementationHandler(sourceImplementationsHandler, schedulerHandler); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationImplementationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationImplementationsHandler.java index 95d4ecae0d06..b84d599970e7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationImplementationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationImplementationsHandler.java @@ -29,19 +29,20 @@ import io.airbyte.api.model.ConnectionRead; import io.airbyte.api.model.ConnectionStatus; import io.airbyte.api.model.ConnectionUpdate; +import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationImplementationCreate; import io.airbyte.api.model.DestinationImplementationIdRequestBody; import io.airbyte.api.model.DestinationImplementationRead; import io.airbyte.api.model.DestinationImplementationReadList; import io.airbyte.api.model.DestinationImplementationUpdate; +import io.airbyte.api.model.DestinationSpecificationRead; import io.airbyte.api.model.WorkspaceIdRequestBody; +import io.airbyte.commons.json.JsonSchemaValidator; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.DestinationConnectionImplementation; -import io.airbyte.config.DestinationConnectionSpecification; import io.airbyte.config.StandardDestination; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.server.validation.IntegrationSchemaValidation; import java.io.IOException; import java.util.List; import java.util.UUID; @@ -49,39 +50,43 @@ public class DestinationImplementationsHandler { - private ConnectionsHandler connectionsHandler; + private final ConnectionsHandler connectionsHandler; + private final SchedulerHandler schedulerHandler; private final Supplier uuidGenerator; private final ConfigRepository configRepository; - private final IntegrationSchemaValidation validator; + private final JsonSchemaValidator validator; public DestinationImplementationsHandler(final ConfigRepository configRepository, - final IntegrationSchemaValidation integrationSchemaValidation, + final JsonSchemaValidator integrationSchemaValidation, + final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler, final Supplier uuidGenerator) { this.configRepository = configRepository; this.validator = integrationSchemaValidation; + this.schedulerHandler = schedulerHandler; this.connectionsHandler = connectionsHandler; this.uuidGenerator = uuidGenerator; } public DestinationImplementationsHandler(final ConfigRepository configRepository, - final IntegrationSchemaValidation integrationSchemaValidation, + final JsonSchemaValidator integrationSchemaValidation, + final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler) { - this(configRepository, integrationSchemaValidation, connectionsHandler, UUID::randomUUID); + this(configRepository, integrationSchemaValidation, schedulerHandler, connectionsHandler, UUID::randomUUID); } public DestinationImplementationRead createDestinationImplementation(final DestinationImplementationCreate destinationImplementationCreate) throws ConfigNotFoundException, IOException, JsonValidationException { // validate configuration validateDestinationImplementation( - destinationImplementationCreate.getDestinationSpecificationId(), + destinationImplementationCreate.getDestinationId(), destinationImplementationCreate.getConnectionConfiguration()); // persist final UUID destinationImplementationId = uuidGenerator.get(); persistDestinationConnectionImplementation( destinationImplementationCreate.getName() != null ? destinationImplementationCreate.getName() : "default", - destinationImplementationCreate.getDestinationSpecificationId(), + destinationImplementationCreate.getDestinationId(), destinationImplementationCreate.getWorkspaceId(), destinationImplementationId, destinationImplementationCreate.getConnectionConfiguration(), @@ -117,7 +122,7 @@ public void deleteDestinationImplementation(final DestinationImplementationIdReq // persist persistDestinationConnectionImplementation( destinationImpl.getName(), - destinationImpl.getDestinationSpecificationId(), + destinationImpl.getDestinationId(), destinationImpl.getWorkspaceId(), destinationImpl.getDestinationImplementationId(), destinationImpl.getConnectionConfiguration(), @@ -132,13 +137,13 @@ public DestinationImplementationRead updateDestinationImplementation(final Desti // validate configuration validateDestinationImplementation( - currentDci.getDestinationSpecificationId(), + currentDci.getDestinationId(), destinationImplementationUpdate.getConnectionConfiguration()); // persist persistDestinationConnectionImplementation( destinationImplementationUpdate.getName(), - currentDci.getDestinationSpecificationId(), + currentDci.getDestinationId(), currentDci.getWorkspaceId(), destinationImplementationUpdate.getDestinationImplementationId(), destinationImplementationUpdate.getConnectionConfiguration(), @@ -172,15 +177,15 @@ public DestinationImplementationReadList listDestinationImplementationsForWorksp return new DestinationImplementationReadList().destinations(reads); } - private void validateDestinationImplementation(final UUID destinationConnectionSpecificationId, + private void validateDestinationImplementation(final UUID destinationId, final JsonNode implementationJson) throws JsonValidationException, IOException, ConfigNotFoundException { - DestinationConnectionSpecification dcs = configRepository.getDestinationConnectionSpecification(destinationConnectionSpecificationId); - validator.validateConfig(dcs, implementationJson); + DestinationSpecificationRead dcs = schedulerHandler.getDestinationSpecification(new DestinationIdRequestBody().destinationId(destinationId)); + validator.validate(dcs.getConnectionSpecification(), implementationJson); } private void persistDestinationConnectionImplementation(final String name, - final UUID destinationSpecificationId, + final UUID destinationId, final UUID workspaceId, final UUID destinationImplementationId, final JsonNode configurationJson, @@ -188,7 +193,7 @@ private void persistDestinationConnectionImplementation(final String name, throws JsonValidationException, IOException { final DestinationConnectionImplementation destinationConnectionImplementation = new DestinationConnectionImplementation() .withName(name) - .withDestinationSpecificationId(destinationSpecificationId) + .withDestinationId(destinationId) .withWorkspaceId(workspaceId) .withDestinationImplementationId(destinationImplementationId) .withConfiguration(configurationJson) @@ -201,9 +206,7 @@ private DestinationImplementationRead buildDestinationImplementationRead(final U throws ConfigNotFoundException, IOException, JsonValidationException { // read configuration from db final DestinationConnectionImplementation dci = configRepository.getDestinationConnectionImplementation(destinationImplementationId); - - final UUID destinationId = configRepository.getDestinationConnectionSpecification(dci.getDestinationSpecificationId()).getDestinationId(); - final StandardDestination standardDestination = configRepository.getStandardDestination(destinationId); + final StandardDestination standardDestination = configRepository.getStandardDestination(dci.getDestinationId()); return buildDestinationImplementationRead(dci, standardDestination); } @@ -213,7 +216,7 @@ private DestinationImplementationRead buildDestinationImplementationRead(final D .destinationId(standardDestination.getDestinationId()) .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) - .destinationSpecificationId(destinationConnectionImplementation.getDestinationSpecificationId()) + .destinationId(destinationConnectionImplementation.getDestinationId()) .connectionConfiguration(destinationConnectionImplementation.getConfiguration()) .name(destinationConnectionImplementation.getName()) .destinationName(standardDestination.getName()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index f1a2008604cd..7f7cd73cf548 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -36,15 +36,14 @@ import io.airbyte.api.model.SourceImplementationDiscoverSchemaRead; import io.airbyte.api.model.SourceImplementationIdRequestBody; import io.airbyte.api.model.SourceSpecificationRead; +import io.airbyte.commons.docker.DockerUtil; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.DestinationConnectionImplementation; -import io.airbyte.config.DestinationConnectionSpecification; import io.airbyte.config.JobOutput; import io.airbyte.config.Schema; import io.airbyte.config.SourceConnectionImplementation; -import io.airbyte.config.SourceConnectionSpecification; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDestination; import io.airbyte.config.StandardDiscoverSchemaOutput; @@ -52,17 +51,13 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.integrations.Integrations; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.persistence.SchedulerPersistence; -import io.airbyte.commons.docker.DockerUtil; import io.airbyte.server.converters.SchemaConverter; - import java.io.IOException; import java.util.Collections; import java.util.UUID; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -156,19 +151,14 @@ public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementat public SourceSpecificationRead getSourceSpecification(SourceIdRequestBody sourceIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { UUID sourceId = sourceIdRequestBody.getSourceId(); - // TODO this will be removed once specs no longer have dedicated IDs - final SourceConnectionSpecification sourceConnectionSpecification = - configRepository.getSourceConnectionSpecificationFromSourceId(sourceId); - - final UUID specId = sourceConnectionSpecification.getSourceSpecificationId(); - final String imageName = Integrations.findBySpecId(specId).getTaggedImage(); + StandardSource source = configRepository.getStandardSource(sourceId); + final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); final long jobId = schedulerPersistence.createGetSpecJob(imageName); LOGGER.debug("getSourceSpec jobId = {}", jobId); Job job = waitUntilJobIsTerminalOrTimeout(jobId); TrackingClientSingleton.get().track("get_source_spec", ImmutableMap.builder() - .put("source_specification_id", specId) .put("source_id", sourceId) .put("image_name", imageName) .put("job_id", jobId) @@ -178,26 +168,20 @@ public SourceSpecificationRead getSourceSpecification(SourceIdRequestBody source return new SourceSpecificationRead() .connectionSpecification(spec.getConnectionSpecification()) .documentationUrl(spec.getDocumentationUrl().toString()) - .sourceId(sourceId) - .sourceSpecificationId(specId); + .sourceId(sourceId); } public DestinationSpecificationRead getDestinationSpecification(DestinationIdRequestBody destinationIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { UUID destinationId = destinationIdRequestBody.getDestinationId(); - // TODO this will be removed once specs no longer have dedicated IDs - final DestinationConnectionSpecification destinationSpec = - configRepository.getDestinationConnectionSpecificationFromDestinationId(destinationId); - - final UUID specId = destinationSpec.getDestinationSpecificationId(); - final String imageName = Integrations.findBySpecId(specId).getTaggedImage(); + StandardDestination destination = configRepository.getStandardDestination(destinationId); + final String imageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final long jobId = schedulerPersistence.createGetSpecJob(imageName); LOGGER.debug("getSourceSpec jobId = {}", jobId); Job job = waitUntilJobIsTerminalOrTimeout(jobId); TrackingClientSingleton.get().track("get_source_spec", ImmutableMap.builder() - .put("destination_specification_id", specId) .put("destination_id", destinationId) .put("image_name", imageName) .put("job_id", jobId) @@ -207,8 +191,7 @@ public DestinationSpecificationRead getDestinationSpecification(DestinationIdReq return new DestinationSpecificationRead() .connectionSpecification(spec.getConnectionSpecification()) .documentationUrl(spec.getDocumentationUrl().toString()) - .destinationId(destinationId) - .destinationSpecificationId(specId); + .destinationId(destinationId); } public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) @@ -224,7 +207,7 @@ public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectio StandardSource source = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); final String sourceImageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); - StandardSource destination = configRepository.getStandardSource(destinationConnectionImplementation.getDestinationId()); + StandardDestination destination = configRepository.getStandardDestination(destinationConnectionImplementation.getDestinationId()); final String destinationImageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final long jobId = schedulerPersistence.createSyncJob( @@ -232,8 +215,7 @@ public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectio destinationConnectionImplementation, standardSync, sourceImageName, - destinationImageName - ); + destinationImageName); final Job job = waitUntilJobIsTerminalOrTimeout(jobId); TrackingClientSingleton.get().track("sync", ImmutableMap.builder() diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java index 2e1f7b822db2..d24f60d6f586 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourceImplementationsHandler.java @@ -29,20 +29,20 @@ import io.airbyte.api.model.ConnectionRead; import io.airbyte.api.model.ConnectionStatus; import io.airbyte.api.model.ConnectionUpdate; +import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceImplementationCreate; import io.airbyte.api.model.SourceImplementationIdRequestBody; import io.airbyte.api.model.SourceImplementationRead; import io.airbyte.api.model.SourceImplementationReadList; import io.airbyte.api.model.SourceImplementationUpdate; +import io.airbyte.api.model.SourceSpecificationRead; import io.airbyte.api.model.WorkspaceIdRequestBody; +import io.airbyte.commons.json.JsonSchemaValidator; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.SourceConnectionImplementation; -import io.airbyte.config.SourceConnectionSpecification; import io.airbyte.config.StandardSource; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.server.validation.IntegrationSchemaValidation; - import java.io.IOException; import java.util.List; import java.util.UUID; @@ -52,12 +52,12 @@ public class SourceImplementationsHandler { private final Supplier uuidGenerator; private final ConfigRepository configRepository; - private final IntegrationSchemaValidation validator; + private final JsonSchemaValidator validator; private SchedulerHandler schedulerHandler; private final ConnectionsHandler connectionsHandler; public SourceImplementationsHandler(final ConfigRepository configRepository, - final IntegrationSchemaValidation integrationSchemaValidation, + final JsonSchemaValidator integrationSchemaValidation, final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler, final Supplier uuidGenerator) { @@ -69,7 +69,7 @@ public SourceImplementationsHandler(final ConfigRepository configRepository, } public SourceImplementationsHandler(final ConfigRepository configRepository, - final IntegrationSchemaValidation integrationSchemaValidation, + final JsonSchemaValidator integrationSchemaValidation, final SchedulerHandler schedulerHandler, final ConnectionsHandler connectionsHandler) { this(configRepository, integrationSchemaValidation, schedulerHandler, connectionsHandler, UUID::randomUUID); @@ -186,10 +186,11 @@ private SourceImplementationRead buildSourceImplementationRead(UUID sourceImplem return toSourceImplementationRead(sourceConnectionImplementation, standardSource); } - private void validateSourceImplementation(UUID sourceConnectionSpecificationId, JsonNode implementationJson) + private void validateSourceImplementation(UUID sourceId, JsonNode implementationJson) throws JsonValidationException, IOException, ConfigNotFoundException { - SourceConnectionSpecification scs = schedulerHandler.getSourceConnectionSpecification(sourceConnectionSpecificationId); - validator.validateConfig(scs, implementationJson); + SourceSpecificationRead scs = schedulerHandler.getSourceSpecification(new SourceIdRequestBody().sourceId(sourceId)); + + validator.validate(scs.getConnectionSpecification(), implementationJson); } private void persistSourceConnectionImplementation(final String name, diff --git a/airbyte-server/src/main/java/io/airbyte/server/validation/IntegrationSchemaValidation.java b/airbyte-server/src/main/java/io/airbyte/server/validation/IntegrationSchemaValidation.java deleted file mode 100644 index 5bf41a590d75..000000000000 --- a/airbyte-server/src/main/java/io/airbyte/server/validation/IntegrationSchemaValidation.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.server.validation; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.JsonSchemaValidator; -import io.airbyte.commons.json.JsonValidationException; -import io.airbyte.config.DestinationConnectionSpecification; -import io.airbyte.config.SourceConnectionSpecification; - -public class IntegrationSchemaValidation { - - private final JsonSchemaValidator jsonSchemaValidator; - - public IntegrationSchemaValidation() { - this.jsonSchemaValidator = new JsonSchemaValidator(); - } - - public void validateConfig(final SourceConnectionSpecification sourceConnectionSpecification, - final JsonNode configJson) - throws JsonValidationException { - final JsonNode schemaJson = sourceConnectionSpecification.getSpecification(); - - jsonSchemaValidator.ensure(schemaJson, configJson); - } - - public void validateConfig(final DestinationConnectionSpecification destinationConnectionSpecification, - final JsonNode configJson) - throws JsonValidationException { - final JsonNode schemaJson = destinationConnectionSpecification.getSpecification(); - - jsonSchemaValidator.ensure(schemaJson, configJson); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationImplementationsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationImplementationsHandlerTest.java index a6e8f9a606f9..8646e1edc479 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationImplementationsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationImplementationsHandlerTest.java @@ -36,25 +36,27 @@ import io.airbyte.api.model.ConnectionReadList; import io.airbyte.api.model.ConnectionStatus; import io.airbyte.api.model.ConnectionUpdate; +import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationImplementationCreate; import io.airbyte.api.model.DestinationImplementationIdRequestBody; import io.airbyte.api.model.DestinationImplementationRead; import io.airbyte.api.model.DestinationImplementationReadList; import io.airbyte.api.model.DestinationImplementationUpdate; +import io.airbyte.api.model.DestinationSpecificationRead; import io.airbyte.api.model.WorkspaceIdRequestBody; +import io.airbyte.commons.json.JsonSchemaValidator; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.DestinationConnectionImplementation; -import io.airbyte.config.DestinationConnectionSpecification; import io.airbyte.config.StandardDestination; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.server.helpers.ConnectionHelpers; +import io.airbyte.server.helpers.ConnectorSpecificationHelpers; import io.airbyte.server.helpers.DestinationHelpers; import io.airbyte.server.helpers.DestinationImplementationHelpers; -import io.airbyte.server.helpers.DestinationSpecificationHelpers; -import io.airbyte.server.validation.IntegrationSchemaValidation; import java.io.IOException; import java.util.Collections; import java.util.UUID; @@ -66,28 +68,36 @@ class DestinationImplementationsHandlerTest { private ConfigRepository configRepository; private StandardDestination standardDestination; - private DestinationConnectionSpecification destinationConnectionSpecification; + private DestinationSpecificationRead destinationSpecificationRead; + private DestinationIdRequestBody destinationIdRequestBody; private DestinationConnectionImplementation destinationConnectionImplementation; private DestinationImplementationsHandler destinationImplementationsHandler; private ConnectionsHandler connectionsHandler; + private SchedulerHandler schedulerHandler; - private IntegrationSchemaValidation validator; + private JsonSchemaValidator validator; private Supplier uuidGenerator; @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException { configRepository = mock(ConfigRepository.class); - validator = mock(IntegrationSchemaValidation.class); + validator = mock(JsonSchemaValidator.class); uuidGenerator = mock(Supplier.class); connectionsHandler = mock(ConnectionsHandler.class); + schedulerHandler = mock(SchedulerHandler.class); standardDestination = DestinationHelpers.generateDestination(); - destinationConnectionSpecification = DestinationSpecificationHelpers.generateDestinationSpecification(standardDestination.getDestinationId()); - destinationConnectionImplementation = DestinationImplementationHelpers.generateDestinationImplementation( - destinationConnectionSpecification.getDestinationSpecificationId()); - - destinationImplementationsHandler = new DestinationImplementationsHandler(configRepository, validator, connectionsHandler, uuidGenerator); + destinationIdRequestBody = new DestinationIdRequestBody().destinationId(standardDestination.getDestinationId()); + ConnectorSpecification connectorSpecification = ConnectorSpecificationHelpers.generateConnectorSpecification(); + destinationSpecificationRead = new DestinationSpecificationRead() + .connectionSpecification(connectorSpecification.getConnectionSpecification()) + .destinationId(standardDestination.getDestinationId()) + .documentationUrl(connectorSpecification.getDocumentationUrl().toString()); + + destinationConnectionImplementation = DestinationImplementationHelpers.generateDestinationImplementation(standardDestination.getDestinationId()); + destinationImplementationsHandler = + new DestinationImplementationsHandler(configRepository, validator, schedulerHandler, connectionsHandler, uuidGenerator); } @Test @@ -99,16 +109,15 @@ void testCreateDestinationImplementation() when(configRepository.getDestinationConnectionImplementation(destinationConnectionImplementation.getDestinationImplementationId())) .thenReturn(destinationConnectionImplementation); - when(configRepository.getDestinationConnectionSpecification(destinationConnectionImplementation.getDestinationSpecificationId())) - .thenReturn(destinationConnectionSpecification); + when(schedulerHandler.getDestinationSpecification(destinationIdRequestBody)).thenReturn(destinationSpecificationRead); - when(configRepository.getStandardDestination(destinationConnectionSpecification.getDestinationId())) + when(configRepository.getStandardDestination(standardDestination.getDestinationId())) .thenReturn(standardDestination); final DestinationImplementationCreate destinationImplementationCreate = new DestinationImplementationCreate() .name(destinationConnectionImplementation.getName()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) - .destinationSpecificationId(destinationConnectionSpecification.getDestinationSpecificationId()) + .destinationId(standardDestination.getDestinationId()) .connectionConfiguration(DestinationImplementationHelpers.getTestImplementationJson()); final DestinationImplementationRead actualDestinationImplementationRead = @@ -116,8 +125,7 @@ void testCreateDestinationImplementation() DestinationImplementationRead expectedDestinationImplementationRead = new DestinationImplementationRead() .name(destinationConnectionImplementation.getName()) - .destinationId(destinationConnectionSpecification.getDestinationId()) - .destinationSpecificationId(destinationConnectionSpecification.getDestinationSpecificationId()) + .destinationId(standardDestination.getDestinationId()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) .connectionConfiguration(DestinationImplementationHelpers.getTestImplementationJson()) @@ -126,8 +134,8 @@ void testCreateDestinationImplementation() assertEquals(expectedDestinationImplementationRead, actualDestinationImplementationRead); verify(validator) - .validateConfig( - destinationConnectionSpecification, + .validate( + destinationSpecificationRead.getConnectionSpecification(), destinationConnectionImplementation.getConfiguration()); verify(configRepository).writeDestinationConnectionImplementation(destinationConnectionImplementation); @@ -145,11 +153,8 @@ void testDeleteDestinationImplementation() throws JsonValidationException, Confi .thenReturn(destinationConnectionImplementation) .thenReturn(expectedDestinationConnectionImplementation); - when(configRepository.getDestinationConnectionSpecification(destinationConnectionSpecification.getDestinationSpecificationId())) - .thenReturn(destinationConnectionSpecification); - - when(configRepository.getStandardDestination(destinationConnectionSpecification.getDestinationId())) - .thenReturn(standardDestination); + when(schedulerHandler.getDestinationSpecification(destinationIdRequestBody)).thenReturn(destinationSpecificationRead); + when(configRepository.getStandardDestination(standardDestination.getDestinationId())).thenReturn(standardDestination); final DestinationImplementationIdRequestBody destinationImplementationId = new DestinationImplementationIdRequestBody() .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()); @@ -194,11 +199,8 @@ void testUpdateDestinationImplementation() .thenReturn(destinationConnectionImplementation) .thenReturn(expectedDestinationConnectionImplementation); - when(configRepository.getDestinationConnectionSpecification(destinationConnectionImplementation.getDestinationSpecificationId())) - .thenReturn(destinationConnectionSpecification); - - when(configRepository.getStandardDestination(destinationConnectionSpecification.getDestinationId())) - .thenReturn(standardDestination); + when(schedulerHandler.getDestinationSpecification(destinationIdRequestBody)).thenReturn(destinationSpecificationRead); + when(configRepository.getStandardDestination(standardDestination.getDestinationId())).thenReturn(standardDestination); final DestinationImplementationUpdate destinationImplementationUpdate = new DestinationImplementationUpdate() .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) @@ -210,8 +212,7 @@ void testUpdateDestinationImplementation() DestinationImplementationRead expectedDestinationImplementationRead = new DestinationImplementationRead() .name(destinationConnectionImplementation.getName()) - .destinationId(destinationConnectionSpecification.getDestinationId()) - .destinationSpecificationId(destinationConnectionSpecification.getDestinationSpecificationId()) + .destinationId(standardDestination.getDestinationId()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) .connectionConfiguration(newConfiguration) @@ -227,16 +228,12 @@ void testGetDestinationImplementation() throws JsonValidationException, ConfigNo when(configRepository.getDestinationConnectionImplementation(destinationConnectionImplementation.getDestinationImplementationId())) .thenReturn(destinationConnectionImplementation); - when(configRepository.getDestinationConnectionSpecification(destinationConnectionImplementation.getDestinationSpecificationId())) - .thenReturn(destinationConnectionSpecification); - - when(configRepository.getStandardDestination(destinationConnectionSpecification.getDestinationId())) - .thenReturn(standardDestination); + when(schedulerHandler.getDestinationSpecification(destinationIdRequestBody)).thenReturn(destinationSpecificationRead); + when(configRepository.getStandardDestination(standardDestination.getDestinationId())).thenReturn(standardDestination); DestinationImplementationRead expectedDestinationImplementationRead = new DestinationImplementationRead() .name(destinationConnectionImplementation.getName()) - .destinationId(destinationConnectionSpecification.getDestinationId()) - .destinationSpecificationId(destinationConnectionImplementation.getDestinationSpecificationId()) + .destinationId(standardDestination.getDestinationId()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) .connectionConfiguration(destinationConnectionImplementation.getConfiguration()) @@ -258,15 +255,12 @@ void testListDestinationImplementationsForWorkspace() .thenReturn(destinationConnectionImplementation); when(configRepository.listDestinationConnectionImplementations()) .thenReturn(Lists.newArrayList(destinationConnectionImplementation)); - when(configRepository.getDestinationConnectionSpecification(destinationConnectionImplementation.getDestinationSpecificationId())) - .thenReturn(destinationConnectionSpecification); - when(configRepository.getStandardDestination(destinationConnectionSpecification.getDestinationId())) - .thenReturn(standardDestination); + when(schedulerHandler.getDestinationSpecification(destinationIdRequestBody)).thenReturn(destinationSpecificationRead); + when(configRepository.getStandardDestination(standardDestination.getDestinationId())).thenReturn(standardDestination); DestinationImplementationRead expectedDestinationImplementationRead = new DestinationImplementationRead() .name(destinationConnectionImplementation.getName()) - .destinationId(destinationConnectionSpecification.getDestinationId()) - .destinationSpecificationId(destinationConnectionImplementation.getDestinationSpecificationId()) + .destinationId(standardDestination.getDestinationId()) .workspaceId(destinationConnectionImplementation.getWorkspaceId()) .destinationImplementationId(destinationConnectionImplementation.getDestinationImplementationId()) .connectionConfiguration(destinationConnectionImplementation.getConfiguration()) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 2a247b62a505..8526e8940c4e 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -37,21 +37,21 @@ import io.airbyte.api.model.DestinationImplementationIdRequestBody; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceImplementationIdRequestBody; +import io.airbyte.commons.docker.DockerUtil; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.DestinationConnectionImplementation; -import io.airbyte.config.DestinationConnectionSpecification; import io.airbyte.config.JobOutput; import io.airbyte.config.SourceConnectionImplementation; -import io.airbyte.config.SourceConnectionSpecification; import io.airbyte.config.StandardCheckConnectionOutput; +import io.airbyte.config.StandardDestination; import io.airbyte.config.StandardGetSpecOutput; +import io.airbyte.config.StandardSource; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.integrations.Integrations; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.persistence.SchedulerPersistence; @@ -70,8 +70,15 @@ class SchedulerHandlerTest { private static final long JOB_ID = 15L; - private static final String SOURCE_IMAGE_NAME = "srcimage"; - private static final String DESTINATION_IMAGE_NAME = "dstimage"; + + private static final String SOURCE_DOCKER_REPO = "srcimage"; + private static final String SOURCE_DOCKER_TAG = "tag"; + private static final String SOURCE_DOCKER_IMAGE = DockerUtil.getTaggedImageName(SOURCE_DOCKER_REPO, SOURCE_DOCKER_TAG); + + private static final String DESTINATION_DOCKER_REPO = "dstimage"; + private static final String DESTINATION_DOCKER_TAG = "tag"; + private static final String DESTINATION_DOCKER_IMAGE = DockerUtil.getTaggedImageName(DESTINATION_DOCKER_REPO, DESTINATION_DOCKER_TAG); + private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; private SchedulerPersistence schedulerPersistence; @@ -96,26 +103,31 @@ void testCheckSourceImplementationConnection() throws JsonValidationException, I final SourceImplementationIdRequestBody request = new SourceImplementationIdRequestBody().sourceImplementationId(sourceImpl.getSourceImplementationId()); + when(configRepository.getStandardSource(sourceImpl.getSourceId())) + .thenReturn(new StandardSource() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSourceId(sourceImpl.getSourceId())); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); - when(schedulerPersistence.createSourceCheckConnectionJob(sourceImpl, SOURCE_IMAGE_NAME)).thenReturn(JOB_ID); + when(schedulerPersistence.createSourceCheckConnectionJob(sourceImpl, SOURCE_DOCKER_IMAGE)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.checkSourceImplementationConnection(request); verify(configRepository).getSourceConnectionImplementation(sourceImpl.getSourceImplementationId()); - verify(schedulerPersistence).createSourceCheckConnectionJob(sourceImpl, SOURCE_IMAGE_NAME); + verify(schedulerPersistence).createSourceCheckConnectionJob(sourceImpl, SOURCE_DOCKER_IMAGE); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @Test void testGetSourceSpec() throws JsonValidationException, IOException, ConfigNotFoundException, URISyntaxException { - // TODO remove dedicated spec IDs once specs no longer have them - UUID sourceId = UUID.randomUUID(); - UUID postgresSpecId = Integrations.POSTGRES_TAP.getSpecId(); - String postgresImage = Integrations.POSTGRES_TAP.getTaggedImage(); - SourceConnectionSpecification spec = new SourceConnectionSpecification().withSourceSpecificationId(postgresSpecId); - when(configRepository.getSourceConnectionSpecificationFromSourceId(sourceId)).thenReturn(spec); - when(schedulerPersistence.createGetSpecJob(postgresImage)).thenReturn(JOB_ID); + SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(UUID.randomUUID()); + when(configRepository.getStandardSource(sourceIdRequestBody.getSourceId())) + .thenReturn(new StandardSource() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSourceId(sourceIdRequestBody.getSourceId())); + when(schedulerPersistence.createGetSpecJob(SOURCE_DOCKER_IMAGE)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); StandardGetSpecOutput specOutput = new StandardGetSpecOutput().withSpecification( @@ -128,22 +140,23 @@ void testGetSourceSpec() throws JsonValidationException, IOException, ConfigNotF when(jobOutput.getGetSpec()).thenReturn(specOutput); when(completedJob.getOutput()).thenReturn(Optional.of(jobOutput)); - schedulerHandler.getSourceSpecification(new SourceIdRequestBody().sourceId(sourceId)); + schedulerHandler.getSourceSpecification(sourceIdRequestBody); - verify(schedulerPersistence).createGetSpecJob(postgresImage); - verify(configRepository).getSourceConnectionSpecificationFromSourceId(sourceId); + verify(configRepository).getStandardSource(sourceIdRequestBody.getSourceId()); + verify(schedulerPersistence).createGetSpecJob(SOURCE_DOCKER_IMAGE); verify(schedulerPersistence, atLeast(2)).getJob(JOB_ID); } @Test void testGetDestinationSpec() throws JsonValidationException, IOException, ConfigNotFoundException, URISyntaxException { - // TODO remove dedicated spec IDs once specs no longer have them - UUID destinationId = UUID.randomUUID(); - UUID postgresSpecId = Integrations.POSTGRES_TARGET.getSpecId(); - String postgresImage = Integrations.POSTGRES_TARGET.getTaggedImage(); - DestinationConnectionSpecification spec = new DestinationConnectionSpecification().withDestinationSpecificationId(postgresSpecId); - when(configRepository.getDestinationConnectionSpecificationFromDestinationId(destinationId)).thenReturn(spec); - when(schedulerPersistence.createGetSpecJob(postgresImage)).thenReturn(JOB_ID); + DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(UUID.randomUUID()); + + when(configRepository.getStandardDestination(destinationIdRequestBody.getDestinationId())) + .thenReturn(new StandardDestination() + .withDockerRepository(DESTINATION_DOCKER_REPO) + .withDockerImageTag(DESTINATION_DOCKER_TAG) + .withDestinationId(destinationIdRequestBody.getDestinationId())); + when(schedulerPersistence.createGetSpecJob(DESTINATION_DOCKER_IMAGE)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); StandardGetSpecOutput specOutput = new StandardGetSpecOutput().withSpecification( @@ -152,14 +165,13 @@ void testGetDestinationSpec() throws JsonValidationException, IOException, Confi .withChangelogUrl(new URI("https://google.com")) .withConnectionSpecification(Jsons.jsonNode(new HashMap<>()))); JobOutput jobOutput = mock(JobOutput.class); - when(jobOutput.getGetSpec()).thenReturn(specOutput); when(completedJob.getOutput()).thenReturn(Optional.of(jobOutput)); - schedulerHandler.getDestinationSpecification(new DestinationIdRequestBody().destinationId(destinationId)); + schedulerHandler.getDestinationSpecification(destinationIdRequestBody); - verify(schedulerPersistence).createGetSpecJob(postgresImage); - verify(configRepository).getDestinationConnectionSpecificationFromDestinationId(destinationId); + verify(configRepository).getStandardDestination(destinationIdRequestBody.getDestinationId()); + verify(schedulerPersistence).createGetSpecJob(DESTINATION_DOCKER_IMAGE); verify(schedulerPersistence, atLeast(2)).getJob(JOB_ID); } @@ -169,14 +181,19 @@ void testCheckDestinationImplementationConnection() throws IOException, JsonVali final DestinationImplementationIdRequestBody request = new DestinationImplementationIdRequestBody().destinationImplementationId(destinationImpl.getDestinationImplementationId()); + when(configRepository.getStandardDestination(destinationImpl.getDestinationId())) + .thenReturn(new StandardDestination() + .withDockerRepository(DESTINATION_DOCKER_REPO) + .withDockerImageTag(DESTINATION_DOCKER_TAG) + .withDestinationId(destinationImpl.getDestinationId())); when(configRepository.getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId())).thenReturn(destinationImpl); - when(schedulerPersistence.createDestinationCheckConnectionJob(destinationImpl, DESTINATION_IMAGE_NAME)).thenReturn(JOB_ID); + when(schedulerPersistence.createDestinationCheckConnectionJob(destinationImpl, DESTINATION_DOCKER_IMAGE)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.checkDestinationImplementationConnection(request); verify(configRepository).getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId()); - verify(schedulerPersistence).createDestinationCheckConnectionJob(destinationImpl, DESTINATION_IMAGE_NAME); + verify(schedulerPersistence).createDestinationCheckConnectionJob(destinationImpl, DESTINATION_DOCKER_IMAGE); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @@ -186,14 +203,19 @@ void testDiscoverSchemaForSourceImplementation() throws IOException, JsonValidat final SourceImplementationIdRequestBody request = new SourceImplementationIdRequestBody().sourceImplementationId(sourceImpl.getSourceImplementationId()); + when(configRepository.getStandardSource(sourceImpl.getSourceId())) + .thenReturn(new StandardSource() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSourceId(sourceImpl.getSourceId())); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); - when(schedulerPersistence.createDiscoverSchemaJob(sourceImpl, SOURCE_IMAGE_NAME)).thenReturn(JOB_ID); + when(schedulerPersistence.createDiscoverSchemaJob(sourceImpl, SOURCE_DOCKER_IMAGE)).thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.discoverSchemaForSourceImplementation(request); verify(configRepository).getSourceConnectionImplementation(sourceImpl.getSourceImplementationId()); - verify(schedulerPersistence).createDiscoverSchemaJob(sourceImpl, SOURCE_IMAGE_NAME); + verify(schedulerPersistence).createDiscoverSchemaJob(sourceImpl, SOURCE_DOCKER_IMAGE); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } @@ -206,10 +228,21 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot DestinationConnectionImplementation destinationImpl = DestinationImplementationHelpers.generateDestinationImplementation(UUID.randomUUID()) .withDestinationImplementationId(standardSync.getDestinationImplementationId()); + when(configRepository.getStandardSource(sourceImpl.getSourceId())) + .thenReturn(new StandardSource() + .withDockerRepository(SOURCE_DOCKER_REPO) + .withDockerImageTag(SOURCE_DOCKER_TAG) + .withSourceId(sourceImpl.getSourceId())); + when(configRepository.getStandardDestination(destinationImpl.getDestinationId())) + .thenReturn(new StandardDestination() + .withDockerRepository(DESTINATION_DOCKER_REPO) + .withDockerImageTag(DESTINATION_DOCKER_TAG) + .withDestinationId(destinationImpl.getDestinationId())); when(configRepository.getStandardSync(standardSync.getConnectionId())).thenReturn(standardSync); when(configRepository.getSourceConnectionImplementation(sourceImpl.getSourceImplementationId())).thenReturn(sourceImpl); when(configRepository.getDestinationConnectionImplementation(destinationImpl.getDestinationImplementationId())).thenReturn(destinationImpl); - when(schedulerPersistence.createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_IMAGE_NAME, DESTINATION_IMAGE_NAME)).thenReturn(JOB_ID); + when(schedulerPersistence.createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_DOCKER_IMAGE, DESTINATION_DOCKER_IMAGE)) + .thenReturn(JOB_ID); when(schedulerPersistence.getJob(JOB_ID)).thenReturn(inProgressJob).thenReturn(completedJob); schedulerHandler.syncConnection(request); @@ -217,7 +250,7 @@ void testSyncConnection() throws JsonValidationException, IOException, ConfigNot verify(configRepository).getStandardSync(standardSync.getConnectionId()); verify(configRepository).getSourceConnectionImplementation(standardSync.getSourceImplementationId()); verify(configRepository).getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()); - verify(schedulerPersistence).createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_IMAGE_NAME, DESTINATION_IMAGE_NAME); + verify(schedulerPersistence).createSyncJob(sourceImpl, destinationImpl, standardSync, SOURCE_DOCKER_IMAGE, DESTINATION_DOCKER_IMAGE); verify(schedulerPersistence, times(2)).getJob(JOB_ID); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceImplementationsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceImplementationsHandlerTest.java index 1744213e3267..2836ddaab24f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceImplementationsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourceImplementationsHandlerTest.java @@ -36,25 +36,27 @@ import io.airbyte.api.model.ConnectionReadList; import io.airbyte.api.model.ConnectionStatus; import io.airbyte.api.model.ConnectionUpdate; +import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceImplementationCreate; import io.airbyte.api.model.SourceImplementationIdRequestBody; import io.airbyte.api.model.SourceImplementationRead; import io.airbyte.api.model.SourceImplementationReadList; import io.airbyte.api.model.SourceImplementationUpdate; +import io.airbyte.api.model.SourceSpecificationRead; import io.airbyte.api.model.WorkspaceIdRequestBody; +import io.airbyte.commons.json.JsonSchemaValidator; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConnectorSpecification; import io.airbyte.config.SourceConnectionImplementation; -import io.airbyte.config.SourceConnectionSpecification; import io.airbyte.config.StandardSource; import io.airbyte.config.StandardSync; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.server.helpers.ConnectionHelpers; +import io.airbyte.server.helpers.ConnectorSpecificationHelpers; import io.airbyte.server.helpers.SourceHelpers; import io.airbyte.server.helpers.SourceImplementationHelpers; -import io.airbyte.server.helpers.SourceSpecificationHelpers; -import io.airbyte.server.validation.IntegrationSchemaValidation; import java.io.IOException; import java.util.Collections; import java.util.UUID; @@ -66,27 +68,36 @@ class SourceImplementationsHandlerTest { private ConfigRepository configRepository; private StandardSource standardSource; - private SourceConnectionSpecification sourceConnectionSpecification; + private SourceSpecificationRead sourceSpecificationRead; + private SourceIdRequestBody sourceIdRequestBody; private SourceConnectionImplementation sourceConnectionImplementation; private SourceImplementationsHandler sourceImplementationsHandler; - private IntegrationSchemaValidation validator; + private JsonSchemaValidator validator; private ConnectionsHandler connectionsHandler; + private SchedulerHandler schedulerHandler; private Supplier uuidGenerator; @SuppressWarnings("unchecked") @BeforeEach void setUp() throws IOException { configRepository = mock(ConfigRepository.class); - validator = mock(IntegrationSchemaValidation.class); + validator = mock(JsonSchemaValidator.class); connectionsHandler = mock(ConnectionsHandler.class); + schedulerHandler = mock(SchedulerHandler.class); uuidGenerator = mock(Supplier.class); standardSource = SourceHelpers.generateSource(); - sourceConnectionSpecification = SourceSpecificationHelpers.generateSourceSpecification(standardSource.getSourceId()); + sourceIdRequestBody = new SourceIdRequestBody().sourceId(standardSource.getSourceId()); + ConnectorSpecification connectorSpecification = ConnectorSpecificationHelpers.generateConnectorSpecification(); + sourceSpecificationRead = new SourceSpecificationRead() + .sourceId(standardSource.getSourceId()) + .connectionSpecification(connectorSpecification.getConnectionSpecification()) + .documentationUrl(connectorSpecification.getDocumentationUrl().toString()); + sourceConnectionImplementation = - SourceImplementationHelpers.generateSourceImplementation(sourceConnectionSpecification.getSourceSpecificationId()); + SourceImplementationHelpers.generateSourceImplementation(standardSource.getSourceId()); - sourceImplementationsHandler = new SourceImplementationsHandler(configRepository, validator, connectionsHandler, uuidGenerator); + sourceImplementationsHandler = new SourceImplementationsHandler(configRepository, validator, schedulerHandler, connectionsHandler, uuidGenerator); } @Test @@ -98,16 +109,15 @@ void testCreateSourceImplementation() when(configRepository.getSourceConnectionImplementation(sourceConnectionImplementation.getSourceImplementationId())) .thenReturn(sourceConnectionImplementation); - when(configRepository.getSourceConnectionSpecification(sourceConnectionSpecification.getSourceSpecificationId())) - .thenReturn(sourceConnectionSpecification); + when(schedulerHandler.getSourceSpecification(sourceIdRequestBody)).thenReturn(sourceSpecificationRead); - when(configRepository.getStandardSource(sourceConnectionSpecification.getSourceId())) + when(configRepository.getStandardSource(sourceSpecificationRead.getSourceId())) .thenReturn(standardSource); final SourceImplementationCreate sourceImplementationCreate = new SourceImplementationCreate() .name(sourceConnectionImplementation.getName()) .workspaceId(sourceConnectionImplementation.getWorkspaceId()) - .sourceSpecificationId(sourceConnectionSpecification.getSourceSpecificationId()) + .sourceId(standardSource.getSourceId()) .connectionConfiguration(SourceImplementationHelpers.getTestImplementationJson()); final SourceImplementationRead actualSourceImplementationRead = @@ -120,8 +130,8 @@ void testCreateSourceImplementation() assertEquals(expectedSourceImplementationRead, actualSourceImplementationRead); verify(validator) - .validateConfig( - sourceConnectionSpecification, + .validate( + sourceSpecificationRead.getConnectionSpecification(), sourceConnectionImplementation.getConfiguration()); verify(configRepository).writeSourceConnectionImplementation(sourceConnectionImplementation); @@ -140,10 +150,9 @@ void testUpdateSourceImplementation() throws JsonValidationException, ConfigNotF .thenReturn(sourceConnectionImplementation) .thenReturn(expectedSourceConnectionImplementation); - when(configRepository.getSourceConnectionSpecification(sourceConnectionSpecification.getSourceSpecificationId())) - .thenReturn(sourceConnectionSpecification); + when(schedulerHandler.getSourceSpecification(sourceIdRequestBody)).thenReturn(sourceSpecificationRead); - when(configRepository.getStandardSource(sourceConnectionSpecification.getSourceId())) + when(configRepository.getStandardSource(sourceSpecificationRead.getSourceId())) .thenReturn(standardSource); final SourceImplementationUpdate sourceImplementationUpdate = new SourceImplementationUpdate() @@ -168,10 +177,10 @@ void testGetSourceImplementation() throws JsonValidationException, ConfigNotFoun when(configRepository.getSourceConnectionImplementation(sourceConnectionImplementation.getSourceImplementationId())) .thenReturn(sourceConnectionImplementation); - when(configRepository.getSourceConnectionSpecification(sourceConnectionSpecification.getSourceSpecificationId())) - .thenReturn(sourceConnectionSpecification); + when(schedulerHandler.getSourceSpecification(sourceIdRequestBody)) + .thenReturn(sourceSpecificationRead); - when(configRepository.getStandardSource(sourceConnectionSpecification.getSourceId())) + when(configRepository.getStandardSource(sourceSpecificationRead.getSourceId())) .thenReturn(standardSource); SourceImplementationRead expectedSourceImplementationRead = @@ -193,9 +202,8 @@ void testListSourceImplementationsForWorkspace() .thenReturn(sourceConnectionImplementation); when(configRepository.listSourceConnectionImplementations()) .thenReturn(Lists.newArrayList(sourceConnectionImplementation)); - when(configRepository.getSourceConnectionSpecification(sourceConnectionSpecification.getSourceSpecificationId())) - .thenReturn(sourceConnectionSpecification); - when(configRepository.getStandardSource(sourceConnectionSpecification.getSourceId())) + when(schedulerHandler.getSourceSpecification(sourceIdRequestBody)).thenReturn(sourceSpecificationRead); + when(configRepository.getStandardSource(sourceSpecificationRead.getSourceId())) .thenReturn(standardSource); SourceImplementationRead expectedSourceImplementationRead = @@ -222,10 +230,9 @@ void testDeleteSourceImplementation() throws JsonValidationException, ConfigNotF .thenReturn(sourceConnectionImplementation) .thenReturn(expectedSourceConnectionImplementation); - when(configRepository.getSourceConnectionSpecification(sourceConnectionSpecification.getSourceSpecificationId())) - .thenReturn(sourceConnectionSpecification); + when(schedulerHandler.getSourceSpecification(sourceIdRequestBody)).thenReturn(sourceSpecificationRead); - when(configRepository.getStandardSource(sourceConnectionSpecification.getSourceId())) + when(configRepository.getStandardSource(sourceSpecificationRead.getSourceId())) .thenReturn(standardSource); final SourceImplementationIdRequestBody sourceImplementationIdRequestBody = new SourceImplementationIdRequestBody() diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendDestinationImplementationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendDestinationImplementationHandlerTest.java index c02692b42288..0346dc701c24 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendDestinationImplementationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendDestinationImplementationHandlerTest.java @@ -75,7 +75,7 @@ public void testCreatesDestinationWhenCheckConnectionSucceeds() throws JsonValid DestinationImplementationCreate destinationImplementationCreate = new DestinationImplementationCreate(); destinationImplementationCreate.setName(destinationImplementationRead.getName()); destinationImplementationCreate.setConnectionConfiguration(destinationImplementationRead.getConnectionConfiguration()); - destinationImplementationCreate.setDestinationSpecificationId(destinationImplementationRead.getDestinationSpecificationId()); + destinationImplementationCreate.setDestinationId(destinationImplementationRead.getDestinationId()); destinationImplementationCreate.setWorkspaceId(destinationImplementationRead.getWorkspaceId()); when(destinationImplementationsHandler.createDestinationImplementation(destinationImplementationCreate)) @@ -101,7 +101,7 @@ public void testDeletesDestinationWhenCheckConnectionFails() throws JsonValidati DestinationImplementationCreate destinationImplementationCreate = new DestinationImplementationCreate(); destinationImplementationCreate.setName(destinationImplementationRead.getName()); destinationImplementationCreate.setConnectionConfiguration(destinationImplementationRead.getConnectionConfiguration()); - destinationImplementationCreate.setDestinationSpecificationId(destinationImplementationRead.getDestinationSpecificationId()); + destinationImplementationCreate.setDestinationId(destinationImplementationRead.getDestinationId()); destinationImplementationCreate.setWorkspaceId(destinationImplementationRead.getWorkspaceId()); when(destinationImplementationsHandler.createDestinationImplementation(destinationImplementationCreate)) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourceImplementationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourceImplementationHandlerTest.java index 8a75f0f7cc64..5f59f4402377 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourceImplementationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendSourceImplementationHandlerTest.java @@ -74,7 +74,7 @@ public void testCreatesSourceWhenCheckConnectionSucceeds() throws JsonValidation SourceImplementationCreate sourceImplementationCreate = new SourceImplementationCreate(); sourceImplementationCreate.setName(sourceImplementationRead.getName()); sourceImplementationCreate.setConnectionConfiguration(sourceImplementationRead.getConnectionConfiguration()); - sourceImplementationCreate.setSourceSpecificationId(sourceImplementationRead.getSourceSpecificationId()); + sourceImplementationCreate.setSourceId(sourceImplementationRead.getSourceId()); sourceImplementationCreate.setWorkspaceId(sourceImplementationRead.getWorkspaceId()); when(sourceImplementationsHandler.createSourceImplementation(sourceImplementationCreate)).thenReturn(sourceImplementationRead); @@ -98,7 +98,6 @@ public void testDeletesSourceWhenCheckConnectionFails() throws JsonValidationExc SourceImplementationCreate sourceImplementationCreate = new SourceImplementationCreate(); sourceImplementationCreate.setName(sourceImplementationRead.getName()); sourceImplementationCreate.setConnectionConfiguration(sourceImplementationRead.getConnectionConfiguration()); - sourceImplementationCreate.setSourceSpecificationId(sourceImplementationRead.getSourceSpecificationId()); sourceImplementationCreate.setWorkspaceId(sourceImplementationRead.getWorkspaceId()); when(sourceImplementationsHandler.createSourceImplementation(sourceImplementationCreate)).thenReturn(sourceImplementationRead); diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceSpecificationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectorSpecificationHelpers.java similarity index 66% rename from airbyte-server/src/test/java/io/airbyte/server/helpers/SourceSpecificationHelpers.java rename to airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectorSpecificationHelpers.java index bca4b03ada43..575a2e0d5b87 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceSpecificationHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectorSpecificationHelpers.java @@ -25,29 +25,27 @@ package io.airbyte.server.helpers; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.SourceConnectionSpecification; +import io.airbyte.config.ConnectorSpecification; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.UUID; -public class SourceSpecificationHelpers { +public class ConnectorSpecificationHelpers { - public static SourceConnectionSpecification generateSourceSpecification() throws IOException { - return generateSourceSpecification(UUID.randomUUID()); - } - - public static SourceConnectionSpecification generateSourceSpecification(UUID sourceId) throws IOException { - final UUID sourceSpecificationId = UUID.randomUUID(); + public static ConnectorSpecification generateConnectorSpecification() throws IOException { final Path path = Paths.get("../airbyte-server/src/test/resources/json/TestSpecification.json"); - return new SourceConnectionSpecification() - .withSourceId(sourceId) - .withSourceSpecificationId(sourceSpecificationId) - .withDocumentationUrl("https://airbyte.io") - .withSpecification(Jsons.deserialize(Files.readString(path))); + try { + return new ConnectorSpecification() + .withDocumentationUrl(new URI("https://airbyte.io")) + .withConnectionSpecification(Jsons.deserialize(Files.readString(path))); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationImplementationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationImplementationHelpers.java index 667e0654f25b..03d0c0d8ca7c 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationImplementationHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationImplementationHelpers.java @@ -44,7 +44,7 @@ public static JsonNode getTestImplementationJson() throws IOException { return Jsons.deserialize(Files.readString(path)); } - public static DestinationConnectionImplementation generateDestinationImplementation(UUID destinationSpecificationId) + public static DestinationConnectionImplementation generateDestinationImplementation(UUID destinationId) throws IOException { final UUID workspaceId = UUID.randomUUID(); final UUID destinationImplementationId = UUID.randomUUID(); @@ -54,7 +54,7 @@ public static DestinationConnectionImplementation generateDestinationImplementat return new DestinationConnectionImplementation() .withName("my db2 instance") .withWorkspaceId(workspaceId) - .withDestinationSpecificationId(destinationSpecificationId) + .withDestinationId(destinationId) .withDestinationImplementationId(destinationImplementationId) .withConfiguration(implementationJson) .withTombstone(false); @@ -66,7 +66,7 @@ public static DestinationImplementationRead getDestinationImplementationRead(Des return new DestinationImplementationRead() .destinationId(standardDestination.getDestinationId()) .workspaceId(destinationImplementation.getWorkspaceId()) - .destinationSpecificationId(destinationImplementation.getDestinationSpecificationId()) + .destinationId(destinationImplementation.getDestinationId()) .destinationImplementationId(destinationImplementation.getDestinationImplementationId()) .connectionConfiguration(destinationImplementation.getConfiguration()) .name(destinationImplementation.getName()) diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationSpecificationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationSpecificationHelpers.java deleted file mode 100644 index 69ffe0ecd714..000000000000 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationSpecificationHelpers.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.server.helpers; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.config.DestinationConnectionSpecification; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.UUID; - -public class DestinationSpecificationHelpers { - - public static DestinationConnectionSpecification generateDestinationSpecification() throws IOException { - return generateDestinationSpecification(UUID.randomUUID()); - } - - public static DestinationConnectionSpecification generateDestinationSpecification(UUID destinationId) - throws IOException { - final UUID destinationSpecificationId = UUID.randomUUID(); - - final Path path = - Paths.get("../airbyte-server/src/test/resources/json/TestSpecification.json"); - - return new DestinationConnectionSpecification() - .withDestinationId(destinationId) - .withDestinationSpecificationId(destinationSpecificationId) - .withDocumentationUrl("https://airbyte.io") - .withSpecification(Jsons.deserialize(Files.readString(path))); - } - -} diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceImplementationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceImplementationHelpers.java index 98a99d65dcfd..3954108c08bb 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceImplementationHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceImplementationHelpers.java @@ -37,7 +37,7 @@ public class SourceImplementationHelpers { - public static SourceConnectionImplementation generateSourceImplementation(UUID sourceSpecificationId) + public static SourceConnectionImplementation generateSourceImplementation(UUID sourceId) throws IOException { final UUID workspaceId = UUID.randomUUID(); final UUID sourceImplementationId = UUID.randomUUID(); @@ -47,7 +47,7 @@ public static SourceConnectionImplementation generateSourceImplementation(UUID s return new SourceConnectionImplementation() .withName("my postgres db") .withWorkspaceId(workspaceId) - .withSourceSpecificationId(sourceSpecificationId) + .withSourceId(sourceId) .withSourceImplementationId(sourceImplementationId) .withConfiguration(implementationJson) .withTombstone(false); @@ -64,7 +64,7 @@ public static SourceImplementationRead getSourceImplementationRead(SourceConnect return new SourceImplementationRead() .sourceId(standardSource.getSourceId()) .workspaceId(sourceImplementation.getWorkspaceId()) - .sourceSpecificationId(sourceImplementation.getSourceSpecificationId()) + .sourceId(sourceImplementation.getSourceId()) .sourceImplementationId(sourceImplementation.getSourceImplementationId()) .connectionConfiguration(sourceImplementation.getConfiguration()) .name(sourceImplementation.getName()) From 8eaa61be0825e53be38b5dcfdc5fb8db1be2d4a9 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 01:19:27 -0700 Subject: [PATCH 05/12] fix acc tests --- .../test/acceptance/AcceptanceTests.java | 69 ++++++++----------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index 0a5f7a2cb42a..b7c50b219ac7 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -24,10 +24,6 @@ package io.airbyte.test.acceptance; -import static io.airbyte.api.client.model.ConnectionSchedule.TimeUnitEnum.MINUTES; -import static java.time.temporal.ChronoUnit.SECONDS; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.google.common.collect.ImmutableMap; import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.ApiClient; @@ -40,31 +36,18 @@ import io.airbyte.api.client.model.ConnectionStatus; import io.airbyte.api.client.model.ConnectionSyncRead; import io.airbyte.api.client.model.ConnectionUpdate; -import io.airbyte.api.client.model.DestinationIdRequestBody; import io.airbyte.api.client.model.DestinationImplementationCreate; import io.airbyte.api.client.model.DestinationImplementationIdRequestBody; import io.airbyte.api.client.model.DestinationImplementationRead; -import io.airbyte.api.client.model.SourceIdRequestBody; import io.airbyte.api.client.model.SourceImplementationCreate; import io.airbyte.api.client.model.SourceImplementationIdRequestBody; import io.airbyte.api.client.model.SourceImplementationRead; import io.airbyte.api.client.model.SourceSchema; -import io.airbyte.api.client.model.SourceSpecificationRead; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.persistence.PersistenceConstants; import io.airbyte.db.DatabaseHelper; import io.airbyte.test.utils.PostgreSQLContainerHelper; -import java.io.IOException; -import java.sql.SQLException; -import java.time.Duration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; import org.apache.commons.compress.utils.Lists; import org.apache.commons.dbcp2.BasicDataSource; import org.jooq.Condition; @@ -80,6 +63,21 @@ import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static io.airbyte.api.client.model.ConnectionSchedule.TimeUnitEnum.MINUTES; +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + @SuppressWarnings("rawtypes") // We order tests so that independent operations are first and operations dependent on them come // last @@ -135,7 +133,7 @@ public void tearDown() throws ApiException { @Order(1) public void testCreateDestinationImpl() throws ApiException { Map destinationDbConfig = getDestinationDbConfig(); - UUID postgresDestinationSpecId = getPostgresDestinationSpecId(); + UUID postgresDestinationSpecId = getPostgresDestinationId(); UUID workspaceId = PersistenceConstants.DEFAULT_WORKSPACE_ID; String name = "AccTestDestinationDb-" + UUID.randomUUID().toString(); @@ -146,7 +144,7 @@ public void testCreateDestinationImpl() throws ApiException { getDestinationDbConfig()); assertEquals(name, destinationImpl.getName()); - assertEquals(postgresDestinationSpecId, destinationImpl.getDestinationSpecificationId()); + assertEquals(postgresDestinationSpecId, destinationImpl.getDestinationId()); assertEquals(workspaceId, destinationImpl.getWorkspaceId()); assertEquals(Jsons.jsonNode(destinationDbConfig), destinationImpl.getConnectionConfiguration()); } @@ -168,19 +166,19 @@ public void testDestinationCheckConnection() throws ApiException { @Order(3) public void testCreateSourceImplementation() throws ApiException { String dbName = "acc-test-db"; - UUID postgresSourceSpecId = getPostgresSourceSpecId(); + UUID postgresSourceId = getPostgresSourceId(); UUID defaultWorkspaceId = PersistenceConstants.DEFAULT_WORKSPACE_ID; Map sourceDbConfig = getSourceDbConfig(); SourceImplementationRead response = createSourceImplementation( dbName, defaultWorkspaceId, - postgresSourceSpecId, + postgresSourceId, sourceDbConfig); assertEquals(dbName, response.getName()); assertEquals(defaultWorkspaceId, response.getWorkspaceId()); - assertEquals(postgresSourceSpecId, response.getSourceSpecificationId()); + assertEquals(postgresSourceId, response.getSourceId()); assertEquals(Jsons.jsonNode(sourceDbConfig), response.getConnectionConfiguration()); } @@ -375,7 +373,7 @@ private DestinationImplementationRead createPostgresDestinationImpl() throws Api return createDestinationImplementation( "AccTestDestination-" + UUID.randomUUID().toString(), PersistenceConstants.DEFAULT_WORKSPACE_ID, - getPostgresDestinationSpecId(), + getPostgresDestinationId(), getDestinationDbConfig()); } @@ -392,7 +390,7 @@ private Map getDestinationDbConfig() { private DestinationImplementationRead createDestinationImplementation(String name, UUID workspaceId, - UUID destinationSpecId, + UUID destinationId, Map destinationConfig) throws ApiException { DestinationImplementationRead destinationImplementation = @@ -400,21 +398,18 @@ private DestinationImplementationRead createDestinationImplementation(String nam .name(name) .connectionConfiguration(Jsons.jsonNode(destinationConfig)) .workspaceId(workspaceId) - .destinationSpecificationId(destinationSpecId)); + .destinationId(destinationId)); destinationImplIds.add(destinationImplementation.getDestinationImplementationId()); return destinationImplementation; } - private UUID getPostgresDestinationSpecId() throws ApiException { - UUID destinationId = apiClient.getDestinationApi().listDestinations().getDestinations() + private UUID getPostgresDestinationId() throws ApiException { + return apiClient.getDestinationApi().listDestinations().getDestinations() .stream() .filter(dr -> dr.getName().toLowerCase().equals("postgres")) .findFirst() .orElseThrow() .getDestinationId(); - return apiClient.getDestinationSpecificationApi() - .getDestinationSpecification(new DestinationIdRequestBody().destinationId(destinationId)) - .getDestinationSpecificationId(); } private Map getSourceDbConfig() { @@ -431,32 +426,28 @@ private SourceImplementationRead createPostgresSourceImpl() throws ApiException return createSourceImplementation( "acceptanceTestDb-" + UUID.randomUUID().toString(), PersistenceConstants.DEFAULT_WORKSPACE_ID, - getPostgresSourceSpecId(), + getPostgresSourceId(), getSourceDbConfig()); } - private SourceImplementationRead createSourceImplementation(String name, UUID workspaceId, UUID sourceSpecId, Map sourceConfig) + private SourceImplementationRead createSourceImplementation(String name, UUID workspaceId, UUID sourceId, Map sourceConfig) throws ApiException { SourceImplementationRead sourceImplementation = apiClient.getSourceImplementationApi().createSourceImplementation(new SourceImplementationCreate() .name(name) - .sourceSpecificationId(sourceSpecId) + .sourceId(sourceId) .workspaceId(workspaceId) .connectionConfiguration(Jsons.jsonNode(sourceConfig))); sourceImplIds.add(sourceImplementation.getSourceImplementationId()); return sourceImplementation; } - private UUID getPostgresSourceSpecId() throws ApiException { - UUID postgresSourceId = apiClient.getSourceApi().listSources().getSources() + private UUID getPostgresSourceId() throws ApiException { + return apiClient.getSourceApi().listSources().getSources() .stream() .filter(sourceRead -> sourceRead.getName().toLowerCase().equals("postgres")) .findFirst() .orElseThrow() .getSourceId(); - - SourceSpecificationRead sourceSpecRead = - apiClient.getSourceSpecificationApi().getSourceSpecification(new SourceIdRequestBody().sourceId(postgresSourceId)); - return sourceSpecRead.getSourceSpecificationId(); } private void deleteSourceImpl(UUID sourceImplId) throws ApiException { From efda1290b038f8bec1592c61ca14eea22e6c772d Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 01:31:45 -0700 Subject: [PATCH 06/12] spotless --- .../test/acceptance/AcceptanceTests.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java index b7c50b219ac7..1bb4465bd6c0 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/AcceptanceTests.java @@ -24,6 +24,10 @@ package io.airbyte.test.acceptance; +import static io.airbyte.api.client.model.ConnectionSchedule.TimeUnitEnum.MINUTES; +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.google.common.collect.ImmutableMap; import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.ApiClient; @@ -48,6 +52,16 @@ import io.airbyte.config.persistence.PersistenceConstants; import io.airbyte.db.DatabaseHelper; import io.airbyte.test.utils.PostgreSQLContainerHelper; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.compress.utils.Lists; import org.apache.commons.dbcp2.BasicDataSource; import org.jooq.Condition; @@ -63,21 +77,6 @@ import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; -import java.io.IOException; -import java.sql.SQLException; -import java.time.Duration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; - -import static io.airbyte.api.client.model.ConnectionSchedule.TimeUnitEnum.MINUTES; -import static java.time.temporal.ChronoUnit.SECONDS; -import static org.junit.jupiter.api.Assertions.assertEquals; - @SuppressWarnings("rawtypes") // We order tests so that independent operations are first and operations dependent on them come // last From 8b561f9caced1fb993646cc52ec93ce16b29292e Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 01:37:58 -0700 Subject: [PATCH 07/12] rename and remove commented out API endpoints --- airbyte-api/src/main/openapi/config.yaml | 65 ------------------- .../resources/types/StandardDestination.yaml | 2 +- 2 files changed, 1 insertion(+), 66 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 80abc8747a05..ba4cdc0d54c9 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -121,48 +121,6 @@ paths: description: Workspace not found "422": $ref: "#/components/responses/InvalidInput" - # /v1/sources/create: - # post: - # tags: - # - source - # summary: Creates a source - # operationId: createSource - # requestBody: - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/SourceCreate" - # responses: - # "200": - # description: Successful operation - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/SourceRead" - # "422": - # $ref: "#/components/responses/InvalidInput" - # /v1/sources/update: - # post: - # tags: - # - source - # summary: Update a source - # operationId: updateSource - # requestBody: - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/SourceUpdate" - # responses: - # "200": - # description: Successful operation - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/SourceRead" - # "404": - # description: Source not found - # "422": - # $ref: "#/components/responses/InvalidInput" /v1/sources/list: post: tags: @@ -379,29 +337,6 @@ paths: description: Source Implementation not found "422": $ref: "#/components/responses/InvalidInput" - # /v1/destinations/update: - # post: - # tags: - # - destination - # summary: Update destination - # operationId: updateDestination - # requestBody: - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/DestinationUpdate" - # required: true - # responses: - # "200": - # description: Successful operation - # content: - # application/json: - # schema: - # $ref: "#/components/schemas/DestinationRead" - # "404": - # description: Destination not found - # "422": - # $ref: "#/components/responses/InvalidInput" /v1/destinations/list: post: tags: diff --git a/airbyte-config/models/src/main/resources/types/StandardDestination.yaml b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml index 0dc8dcf2082d..2234f989bcda 100644 --- a/airbyte-config/models/src/main/resources/types/StandardDestination.yaml +++ b/airbyte-config/models/src/main/resources/types/StandardDestination.yaml @@ -1,6 +1,6 @@ --- "$schema": http://json-schema.org/draft-07/schema# -"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/Destination.yaml +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardDestination.yaml title: Source description: describes a destination type: object From e934bc17cb12215e1b24a37e04f13b05b111a04e Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 10:55:50 -0700 Subject: [PATCH 08/12] rename dockerutils --- .../commons/docker/{DockerUtil.java => DockerUtils.java} | 2 +- .../docker/{DockerUtilTest.java => DockerUtilsTest.java} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename airbyte-commons/src/main/java/io/airbyte/commons/docker/{DockerUtil.java => DockerUtils.java} (97%) rename airbyte-commons/src/test/java/io/airbyte/commons/docker/{DockerUtilTest.java => DockerUtilsTest.java} (92%) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java similarity index 97% rename from airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java rename to airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java index c5b22e6be249..3f29b82e3bec 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtil.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/docker/DockerUtils.java @@ -24,7 +24,7 @@ package io.airbyte.commons.docker; -public class DockerUtil { +public class DockerUtils { public static String getTaggedImageName(String dockerRepository, String tag) { return String.join(":", dockerRepository, tag); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java similarity index 92% rename from airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java rename to airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java index 703ee6defaad..43fa690c4c14 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/docker/DockerUtilsTest.java @@ -28,13 +28,13 @@ import org.junit.jupiter.api.Test; -class DockerUtilTest { +class DockerUtilsTest { @Test void testGetTaggedImageName() { String repository = "airbyte/repo"; String tag = "12.3"; - assertEquals("airbyte/repo:12.3", DockerUtil.getTaggedImageName(repository, tag)); + assertEquals("airbyte/repo:12.3", DockerUtils.getTaggedImageName(repository, tag)); } } From 61bacdd2f4e0fd9f31e5aae941065ba09ef6ebc3 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 10:57:13 -0700 Subject: [PATCH 09/12] rename dockerutils everywhere --- .../job_factory/DefaultSyncJobFactory.java | 6 ++-- .../DefaultSyncJobFactoryTest.java | 6 ++-- .../DefaultSchedulerPersistenceTest.java | 32 +++++++++---------- .../server/handlers/DebugInfoHandler.java | 2 ++ .../server/handlers/SchedulerHandler.java | 16 +++++----- .../server/handlers/SchedulerHandlerTest.java | 6 ++-- .../server/helpers/DestinationHelpers.java | 5 ++- .../airbyte/server/helpers/SourceHelpers.java | 5 ++- 8 files changed, 43 insertions(+), 35 deletions(-) diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java index 39ce920cf9f1..a4e67cdaf797 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactory.java @@ -24,7 +24,7 @@ package io.airbyte.scheduler.job_factory; -import io.airbyte.commons.docker.DockerUtil; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.DestinationConnectionImplementation; import io.airbyte.config.SourceConnectionImplementation; @@ -60,8 +60,8 @@ public Long create(final UUID connectionId) { final StandardSource source = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); final StandardDestination destination = configRepository.getStandardDestination(destinationConnectionImplementation.getDestinationId()); - final String sourceImageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); - final String destinationImageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String sourceImageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String destinationImageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); return schedulerPersistence.createSyncJob( sourceConnectionImplementation, diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java index 3fcb0ce2f1d9..b56ba912be54 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/job_factory/DefaultSyncJobFactoryTest.java @@ -29,7 +29,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.airbyte.commons.docker.DockerUtil; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.DestinationConnectionImplementation; import io.airbyte.config.SourceConnectionImplementation; @@ -65,11 +65,11 @@ void createSyncJobFromConnectionId() throws JsonValidationException, ConfigNotFo new DestinationConnectionImplementation().withDestinationId(destinationId); final String srcDockerRepo = "srcrepo"; final String srcDockerTag = "tag"; - final String srcDockerImage = DockerUtil.getTaggedImageName(srcDockerRepo, srcDockerTag); + final String srcDockerImage = DockerUtils.getTaggedImageName(srcDockerRepo, srcDockerTag); final String dstDockerRepo = "dstrepo"; final String dstDockerTag = "tag"; - final String dstDockerImage = DockerUtil.getTaggedImageName(dstDockerRepo, dstDockerTag); + final String dstDockerImage = DockerUtils.getTaggedImageName(dstDockerRepo, dstDockerTag); when(configRepository.getStandardSync(connectionId)).thenReturn(standardSync); when(configRepository.getSourceConnectionImplementation(sourceImplId)).thenReturn(sourceConnectionImplementation); diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java index 8bf52af8e54b..74bc891488de 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java @@ -24,13 +24,6 @@ package io.airbyte.scheduler.persistence; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -49,10 +42,18 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.Stream; import io.airbyte.db.DatabaseHelper; -import io.airbyte.integrations.Integrations; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.ScopeHelper; +import org.apache.commons.dbcp2.BasicDataSource; +import org.jooq.Record; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.MountableFile; + import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; @@ -63,14 +64,13 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; -import org.apache.commons.dbcp2.BasicDataSource; -import org.jooq.Record; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class DefaultSchedulerPersistenceTest { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java index b1433afe84a8..b822647a17ed 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java @@ -28,6 +28,8 @@ import com.google.common.collect.Lists; import io.airbyte.api.model.DebugRead; +import io.airbyte.commons.docker.DockerUtils; +import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.integrations.Integrations; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 7f7cd73cf548..234067215dc3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -36,7 +36,7 @@ import io.airbyte.api.model.SourceImplementationDiscoverSchemaRead; import io.airbyte.api.model.SourceImplementationIdRequestBody; import io.airbyte.api.model.SourceSpecificationRead; -import io.airbyte.commons.docker.DockerUtil; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.ConnectorSpecification; @@ -81,7 +81,7 @@ public CheckConnectionRead checkSourceImplementationConnection(SourceImplementat configRepository.getSourceConnectionImplementation(sourceImplementationIdRequestBody.getSourceImplementationId()); final StandardSource source = configRepository.getStandardSource(connectionImplementation.getSourceId()); - final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String imageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); final long jobId = schedulerPersistence.createSourceCheckConnectionJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); @@ -104,7 +104,7 @@ public CheckConnectionRead checkDestinationImplementationConnection(DestinationI configRepository.getDestinationConnectionImplementation(destinationImplementationIdRequestBody.getDestinationImplementationId()); final StandardDestination destination = configRepository.getStandardDestination(connectionImplementation.getDestinationId()); - final String imageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); @@ -127,7 +127,7 @@ public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementat configRepository.getSourceConnectionImplementation(sourceImplementationIdRequestBody.getSourceImplementationId()); StandardSource source = configRepository.getStandardSource(connectionImplementation.getSourceId()); - final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String imageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); final long jobId = schedulerPersistence.createDiscoverSchemaJob(connectionImplementation, imageName); LOGGER.debug("jobId = " + jobId); final Job job = waitUntilJobIsTerminalOrTimeout(jobId); @@ -152,7 +152,7 @@ public SourceSpecificationRead getSourceSpecification(SourceIdRequestBody source throws ConfigNotFoundException, IOException, JsonValidationException { UUID sourceId = sourceIdRequestBody.getSourceId(); StandardSource source = configRepository.getStandardSource(sourceId); - final String imageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String imageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); final long jobId = schedulerPersistence.createGetSpecJob(imageName); LOGGER.debug("getSourceSpec jobId = {}", jobId); @@ -175,7 +175,7 @@ public DestinationSpecificationRead getDestinationSpecification(DestinationIdReq throws ConfigNotFoundException, IOException, JsonValidationException { UUID destinationId = destinationIdRequestBody.getDestinationId(); StandardDestination destination = configRepository.getStandardDestination(destinationId); - final String imageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final long jobId = schedulerPersistence.createGetSpecJob(imageName); LOGGER.debug("getSourceSpec jobId = {}", jobId); @@ -205,10 +205,10 @@ public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectio configRepository.getDestinationConnectionImplementation(standardSync.getDestinationImplementationId()); StandardSource source = configRepository.getStandardSource(sourceConnectionImplementation.getSourceId()); - final String sourceImageName = DockerUtil.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); + final String sourceImageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); StandardDestination destination = configRepository.getStandardDestination(destinationConnectionImplementation.getDestinationId()); - final String destinationImageName = DockerUtil.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); + final String destinationImageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final long jobId = schedulerPersistence.createSyncJob( sourceConnectionImplementation, diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index 8526e8940c4e..2064e4d80a26 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -37,7 +37,7 @@ import io.airbyte.api.model.DestinationImplementationIdRequestBody; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceImplementationIdRequestBody; -import io.airbyte.commons.docker.DockerUtil; +import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.commons.json.Jsons; @@ -73,11 +73,11 @@ class SchedulerHandlerTest { private static final String SOURCE_DOCKER_REPO = "srcimage"; private static final String SOURCE_DOCKER_TAG = "tag"; - private static final String SOURCE_DOCKER_IMAGE = DockerUtil.getTaggedImageName(SOURCE_DOCKER_REPO, SOURCE_DOCKER_TAG); + private static final String SOURCE_DOCKER_IMAGE = DockerUtils.getTaggedImageName(SOURCE_DOCKER_REPO, SOURCE_DOCKER_TAG); private static final String DESTINATION_DOCKER_REPO = "dstimage"; private static final String DESTINATION_DOCKER_TAG = "tag"; - private static final String DESTINATION_DOCKER_IMAGE = DockerUtil.getTaggedImageName(DESTINATION_DOCKER_REPO, DESTINATION_DOCKER_TAG); + private static final String DESTINATION_DOCKER_IMAGE = DockerUtils.getTaggedImageName(DESTINATION_DOCKER_REPO, DESTINATION_DOCKER_TAG); private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java index 0f336821c1ff..243e2dd13fa7 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/DestinationHelpers.java @@ -32,7 +32,10 @@ public class DestinationHelpers { public static StandardDestination generateDestination() { return new StandardDestination() .withDestinationId(UUID.randomUUID()) - .withName("db2"); + .withName("db2") + .withDockerRepository("thebestrepo") + .withDockerImageTag("thelatesttag") + .withDocumentationUrl("https://wikipedia.org"); } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java index 22a2875f9037..a089d07c4711 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java +++ b/airbyte-server/src/test/java/io/airbyte/server/helpers/SourceHelpers.java @@ -32,7 +32,10 @@ public class SourceHelpers { public static StandardSource generateSource() { return new StandardSource() .withSourceId(UUID.randomUUID()) - .withName("marketo"); + .withName("marketo") + .withDockerRepository("thebestrepo") + .withDockerImageTag("thelatesttag") + .withDocumentationUrl("https://wikipedia.org"); } } From 5993d4f09017a6dbccb34e099abd9221f6e85f6b Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 11:17:56 -0700 Subject: [PATCH 10/12] remove dep on Integrations.java --- .../persistence/DefaultSchedulerPersistenceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java index 74bc891488de..b2cac136638e 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java @@ -107,11 +107,11 @@ class DefaultSchedulerPersistenceTest { .withTombstone(false); final UUID destinationImplementationId = UUID.randomUUID(); - final UUID destinationSpecificationId = Integrations.POSTGRES_TARGET.getSpecId(); + final UUID destinationId = UUID.randomUUID(); DESTINATION_CONNECTION_IMPLEMENTATION = new DestinationConnectionImplementation() .withWorkspaceId(workspaceId) - .withDestinationId(destinationSpecificationId) + .withDestinationId(destinationId) .withDestinationImplementationId(destinationImplementationId) .withConfiguration(implementationJson) .withTombstone(false); From b6e0f1329b9979e538d06b3738d4da77ef27836e Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Fri, 9 Oct 2020 11:19:07 -0700 Subject: [PATCH 11/12] dont use unimported classes --- .../DefaultSchedulerPersistenceTest.java | 31 +++++++++---------- .../server/handlers/DebugInfoHandler.java | 2 -- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java index b2cac136638e..cbd0d5a9c288 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/persistence/DefaultSchedulerPersistenceTest.java @@ -24,6 +24,13 @@ package io.airbyte.scheduler.persistence; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -45,15 +52,6 @@ import io.airbyte.scheduler.Job; import io.airbyte.scheduler.JobStatus; import io.airbyte.scheduler.ScopeHelper; -import org.apache.commons.dbcp2.BasicDataSource; -import org.jooq.Record; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; - import java.io.IOException; import java.nio.file.Path; import java.sql.SQLException; @@ -64,13 +62,14 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Supplier; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import org.apache.commons.dbcp2.BasicDataSource; +import org.jooq.Record; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.MountableFile; class DefaultSchedulerPersistenceTest { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java index b822647a17ed..b1433afe84a8 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java @@ -28,8 +28,6 @@ import com.google.common.collect.Lists; import io.airbyte.api.model.DebugRead; -import io.airbyte.commons.docker.DockerUtils; -import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.integrations.Integrations; import java.io.IOException; import java.nio.charset.StandardCharsets; From 371962f16c3e9c1b56a74516d4a36b66606bd34a Mon Sep 17 00:00:00 2001 From: "Sherif A. Nada" Date: Fri, 9 Oct 2020 13:15:07 -0700 Subject: [PATCH 12/12] Add source create/update and destination update endpoints (#529) --- airbyte-api/src/main/openapi/config.yaml | 80 ++++++++++++++++++- .../config/persistence/ConfigRepository.java | 14 ++++ .../airbyte/server/apis/ConfigurationApi.java | 23 +++++- .../server/handlers/DestinationsHandler.java | 31 +++++-- .../server/handlers/SourcesHandler.java | 60 ++++++++++++-- .../handlers/DestinationsHandlerTest.java | 44 ++++++++-- .../server/handlers/SourcesHandlerTest.java | 70 ++++++++++++++-- 7 files changed, 290 insertions(+), 32 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index ba4cdc0d54c9..556ff2695599 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -121,6 +121,48 @@ paths: description: Workspace not found "422": $ref: "#/components/responses/InvalidInput" + /v1/sources/create: + post: + tags: + - source + summary: Creates a source + operationId: createSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceCreate" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceRead" + "422": + $ref: "#/components/responses/InvalidInput" + /v1/sources/update: + post: + tags: + - source + summary: Update a source + operationId: updateSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceUpdate" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceRead" + "404": + description: Source not found + "422": + $ref: "#/components/responses/InvalidInput" /v1/sources/list: post: tags: @@ -337,6 +379,29 @@ paths: description: Source Implementation not found "422": $ref: "#/components/responses/InvalidInput" + /v1/destinations/update: + post: + tags: + - destination + summary: Update destination + operationId: updateDestination + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationUpdate" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationRead" + "404": + description: Destination not found + "422": + $ref: "#/components/responses/InvalidInput" /v1/destinations/list: post: tags: @@ -881,6 +946,7 @@ components: - name - dockerRepository - dockerImageTag + - documentationUrl properties: name: type: string @@ -888,6 +954,9 @@ components: type: string dockerImageTag: type: string + documentationUrl: + type: string + format: uri SourceUpdate: type: object description: Update the source. Currently, the only allowed attribute to update is the default docker image version. @@ -905,7 +974,7 @@ components: - sourceId - name - dockerRepository - - dockerImageVersion + - dockerImageTag properties: sourceId: $ref: "#/components/schemas/SourceId" @@ -913,8 +982,11 @@ components: type: string dockerRepository: type: string - dockerImageVersion: + dockerImageTag: type: string + documentationUrl: + type: string + format: uri SourceReadList: type: object required: @@ -1049,6 +1121,7 @@ components: - name - dockerRepository - dockerImageTag + - documentationUrl properties: destinationId: $ref: "#/components/schemas/DestinationId" @@ -1058,6 +1131,9 @@ components: type: string dockerImageTag: type: string + documentationUrl: + type: string + format: uri DestinationReadList: type: object required: diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index fb3c97940f13..46e3d2f43a2c 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -76,6 +76,13 @@ public List listStandardSources() return persistence.listConfigs(ConfigSchema.STANDARD_SOURCE, StandardSource.class); } + public void writeStandardSource(final StandardSource source) throws JsonValidationException, IOException { + persistence.writeConfig( + ConfigSchema.STANDARD_SOURCE, + source.getSourceId().toString(), + source); + } + public StandardDestination getStandardDestination(final UUID destinationId) throws JsonValidationException, IOException, ConfigNotFoundException { return persistence.getConfig( @@ -89,6 +96,13 @@ public List listStandardDestinations() return persistence.listConfigs(ConfigSchema.STANDARD_DESTINATION, StandardDestination.class); } + public void writeStandardDestination(final StandardDestination destination) throws JsonValidationException, IOException { + persistence.writeConfig( + ConfigSchema.STANDARD_DESTINATION, + destination.getDestinationId().toString(), + destination); + } + public SourceConnectionImplementation getSourceConnectionImplementation(final UUID sourceImplementationId) throws JsonValidationException, IOException, ConfigNotFoundException { return persistence.getConfig( diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 533e708059d3..2b76e851e3e1 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -41,11 +41,13 @@ import io.airbyte.api.model.DestinationRead; import io.airbyte.api.model.DestinationReadList; import io.airbyte.api.model.DestinationSpecificationRead; +import io.airbyte.api.model.DestinationUpdate; import io.airbyte.api.model.JobIdRequestBody; import io.airbyte.api.model.JobInfoRead; import io.airbyte.api.model.JobListRequestBody; import io.airbyte.api.model.JobReadList; import io.airbyte.api.model.SlugRequestBody; +import io.airbyte.api.model.SourceCreate; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceImplementationCreate; import io.airbyte.api.model.SourceImplementationDiscoverSchemaRead; @@ -56,6 +58,7 @@ import io.airbyte.api.model.SourceRead; import io.airbyte.api.model.SourceReadList; import io.airbyte.api.model.SourceSpecificationRead; +import io.airbyte.api.model.SourceUpdate; import io.airbyte.api.model.WbConnectionRead; import io.airbyte.api.model.WbConnectionReadList; import io.airbyte.api.model.WorkspaceIdRequestBody; @@ -156,14 +159,24 @@ public SourceRead getSource(@Valid SourceIdRequestBody sourceIdRequestBody) { return execute(() -> sourcesHandler.getSource(sourceIdRequestBody)); } + @Override + public SourceRead createSource(@Valid SourceCreate sourceCreate) { + return execute(() -> sourcesHandler.createSource(sourceCreate)); + } + + @Override + public SourceRead updateSource(@Valid SourceUpdate sourceUpdate) { + return execute(() -> sourcesHandler.updateSource(sourceUpdate)); + } + // SOURCE SPECIFICATION @Override public SourceSpecificationRead getSourceSpecification(@Valid SourceIdRequestBody sourceIdRequestBody) { return execute(() -> schedulerHandler.getSourceSpecification(sourceIdRequestBody)); } - // SOURCE IMPLEMENTATION + @Override public SourceImplementationRead createSourceImplementation(@Valid SourceImplementationCreate sourceImplementationCreate) { return execute(() -> sourceImplementationsHandler.createSourceImplementation(sourceImplementationCreate)); @@ -201,8 +214,8 @@ public CheckConnectionRead checkConnectionToSourceImplementation(@Valid SourceIm public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(@Valid SourceImplementationIdRequestBody sourceImplementationIdRequestBody) { return execute(() -> schedulerHandler.discoverSchemaForSourceImplementation(sourceImplementationIdRequestBody)); } - // DESTINATION + @Override public DestinationReadList listDestinations() { return execute(destinationsHandler::listDestinations); @@ -213,7 +226,13 @@ public DestinationRead getDestination(@Valid DestinationIdRequestBody destinatio return execute(() -> destinationsHandler.getDestination(destinationIdRequestBody)); } + @Override + public DestinationRead updateDestination(@Valid DestinationUpdate destinationUpdate) { + return execute(() -> destinationsHandler.updateDestination(destinationUpdate)); + } + // DESTINATION SPECIFICATION + @Override public DestinationSpecificationRead getDestinationSpecification(@Valid DestinationIdRequestBody destinationIdRequestBody) { return execute(() -> schedulerHandler.getDestinationSpecification(destinationIdRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationsHandler.java index b1723c49a664..cbc6771fa961 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationsHandler.java @@ -27,11 +27,14 @@ import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationRead; import io.airbyte.api.model.DestinationReadList; +import io.airbyte.api.model.DestinationUpdate; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.StandardDestination; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; import java.util.stream.Collectors; @@ -58,12 +61,30 @@ public DestinationRead getDestination(DestinationIdRequestBody destinationIdRequ return buildDestinationRead(configRepository.getStandardDestination(destinationIdRequestBody.getDestinationId())); } - private static DestinationRead buildDestinationRead(StandardDestination standardDestination) { - final DestinationRead destinationRead = new DestinationRead(); - destinationRead.setDestinationId(standardDestination.getDestinationId()); - destinationRead.setName(standardDestination.getName()); + public DestinationRead updateDestination(DestinationUpdate destinationUpdate) throws ConfigNotFoundException, IOException, JsonValidationException { + StandardDestination currentDestination = configRepository.getStandardDestination(destinationUpdate.getDestinationId()); + StandardDestination newDestination = new StandardDestination() + .withDestinationId(currentDestination.getDestinationId()) + .withDockerImageTag(destinationUpdate.getDockerImageTag()) + .withDockerRepository(currentDestination.getDockerRepository()) + .withName(currentDestination.getName()) + .withDocumentationUrl(currentDestination.getDocumentationUrl()); + + configRepository.writeStandardDestination(newDestination); + return buildDestinationRead(newDestination); + } - return destinationRead; + private static DestinationRead buildDestinationRead(StandardDestination standardDestination) { + try { + return new DestinationRead() + .destinationId(standardDestination.getDestinationId()) + .name(standardDestination.getName()) + .dockerRepository(standardDestination.getDockerRepository()) + .dockerImageTag(standardDestination.getDockerImageTag()) + .documentationUrl(new URI(standardDestination.getDocumentationUrl())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourcesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourcesHandler.java index 6e6176b29f12..256ee86e7b63 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SourcesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SourcesHandler.java @@ -24,23 +24,35 @@ package io.airbyte.server.handlers; +import io.airbyte.api.model.SourceCreate; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceRead; import io.airbyte.api.model.SourceReadList; +import io.airbyte.api.model.SourceUpdate; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.StandardSource; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; +import java.util.UUID; +import java.util.function.Supplier; import java.util.stream.Collectors; public class SourcesHandler { private final ConfigRepository configRepository; + private final Supplier uuidSupplier; public SourcesHandler(final ConfigRepository configRepository) { + this(configRepository, UUID::randomUUID); + } + + public SourcesHandler(final ConfigRepository configRepository, Supplier uuidSupplier) { this.configRepository = configRepository; + this.uuidSupplier = uuidSupplier; } public SourceReadList listSources() throws ConfigNotFoundException, IOException, JsonValidationException { @@ -48,21 +60,53 @@ public SourceReadList listSources() throws ConfigNotFoundException, IOException, .stream() .map(SourcesHandler::buildSourceRead) .collect(Collectors.toList()); - return new SourceReadList().sources(reads); } public SourceRead getSource(SourceIdRequestBody sourceIdRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { - final StandardSource standardSource = configRepository.getStandardSource(sourceIdRequestBody.getSourceId()); - return buildSourceRead(standardSource); + return buildSourceRead(configRepository.getStandardSource(sourceIdRequestBody.getSourceId())); } - private static SourceRead buildSourceRead(StandardSource standardSource) { - final SourceRead sourceRead = new SourceRead(); - sourceRead.setSourceId(standardSource.getSourceId()); - sourceRead.setName(standardSource.getName()); + public SourceRead createSource(SourceCreate sourceCreate) throws JsonValidationException, IOException, ConfigNotFoundException { + // TODO add validation for the incoming docker image + UUID id = uuidSupplier.get(); + StandardSource source = new StandardSource() + .withSourceId(id) + .withDockerRepository(sourceCreate.getDockerRepository()) + .withDockerImageTag(sourceCreate.getDockerImageTag()) + .withDocumentationUrl(sourceCreate.getDocumentationUrl().toString()) + .withName(sourceCreate.getName()); + + configRepository.writeStandardSource(source); - return sourceRead; + return buildSourceRead(source); + } + + public SourceRead updateSource(SourceUpdate sourceUpdate) throws ConfigNotFoundException, IOException, JsonValidationException { + // TODO add validation to ensure the incoming version exists + StandardSource currentSource = configRepository.getStandardSource(sourceUpdate.getSourceId()); + StandardSource newSource = new StandardSource() + .withSourceId(currentSource.getSourceId()) + .withDockerImageTag(sourceUpdate.getDockerImageTag()) + .withDockerRepository(currentSource.getDockerRepository()) + .withDocumentationUrl(currentSource.getDocumentationUrl()) + .withName(currentSource.getName()); + + configRepository.writeStandardSource(newSource); + return buildSourceRead(newSource); + } + + private static SourceRead buildSourceRead(StandardSource standardSource) { + try { + return new SourceRead() + .sourceId(standardSource.getSourceId()) + .name(standardSource.getName()) + .dockerRepository(standardSource.getDockerRepository()) + .dockerImageTag(standardSource.getDockerImageTag()) + .documentationUrl(new URI(standardSource.getDocumentationUrl())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationsHandlerTest.java index ba6f4382b4bc..e51f0a1205a6 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationsHandlerTest.java @@ -25,6 +25,7 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -32,11 +33,14 @@ import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationRead; import io.airbyte.api.model.DestinationReadList; +import io.airbyte.api.model.DestinationUpdate; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.StandardDestination; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,23 +63,31 @@ private StandardDestination generateDestination() { return new StandardDestination() .withDestinationId(destinationId) - .withName("presto"); + .withName("presto") + .withDockerImageTag("12.3") + .withDockerRepository("repo") + .withDocumentationUrl("https://hulu.com"); } @Test - void testListDestinations() throws JsonValidationException, IOException, ConfigNotFoundException { + void testListDestinations() throws JsonValidationException, IOException, ConfigNotFoundException, URISyntaxException { final StandardDestination destination2 = generateDestination(); - when(configRepository.listStandardDestinations()) - .thenReturn(Lists.newArrayList(destination, destination2)); + when(configRepository.listStandardDestinations()).thenReturn(Lists.newArrayList(destination, destination2)); DestinationRead expectedDestinationRead1 = new DestinationRead() .destinationId(destination.getDestinationId()) - .name(destination.getName()); + .name(destination.getName()) + .dockerRepository(destination.getDockerRepository()) + .dockerImageTag(destination.getDockerImageTag()) + .documentationUrl(new URI(destination.getDocumentationUrl())); DestinationRead expectedDestinationRead2 = new DestinationRead() .destinationId(destination2.getDestinationId()) - .name(destination2.getName()); + .name(destination2.getName()) + .dockerRepository(destination2.getDockerRepository()) + .dockerImageTag(destination2.getDockerImageTag()) + .documentationUrl(new URI(destination2.getDocumentationUrl())); final DestinationReadList actualDestinationReadList = destinationHandler.listDestinations(); @@ -85,13 +97,16 @@ void testListDestinations() throws JsonValidationException, IOException, ConfigN } @Test - void testGetDestination() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetDestination() throws JsonValidationException, ConfigNotFoundException, IOException, URISyntaxException { when(configRepository.getStandardDestination(destination.getDestinationId())) .thenReturn(destination); DestinationRead expectedDestinationRead = new DestinationRead() .destinationId(destination.getDestinationId()) - .name(destination.getName()); + .name(destination.getName()) + .dockerRepository(destination.getDockerRepository()) + .dockerImageTag(destination.getDockerImageTag()) + .documentationUrl(new URI(destination.getDocumentationUrl())); final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody() .destinationId(destination.getDestinationId()); @@ -101,4 +116,17 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio assertEquals(expectedDestinationRead, actualDestinationRead); } + @Test + void testUpdateDestination() throws ConfigNotFoundException, IOException, JsonValidationException { + when(configRepository.getStandardDestination(destination.getDestinationId())).thenReturn(destination); + final String newDockerImageTag = "averydifferenttag"; + String currentTag = destinationHandler.getDestination( + new DestinationIdRequestBody().destinationId(destination.getDestinationId())).getDockerImageTag(); + assertNotEquals(newDockerImageTag, currentTag); + + DestinationRead sourceRead = destinationHandler.updateDestination( + new DestinationUpdate().destinationId(destination.getDestinationId()).dockerImageTag(newDockerImageTag)); + assertEquals(newDockerImageTag, sourceRead.getDockerImageTag()); + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourcesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourcesHandlerTest.java index 0600e430e57d..7a0bfdcdcaca 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SourcesHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SourcesHandlerTest.java @@ -25,19 +25,25 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.common.collect.Lists; +import io.airbyte.api.model.SourceCreate; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceRead; import io.airbyte.api.model.SourceReadList; +import io.airbyte.api.model.SourceUpdate; import io.airbyte.commons.json.JsonValidationException; import io.airbyte.config.StandardSource; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.UUID; +import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -46,12 +52,16 @@ class SourcesHandlerTest { private ConfigRepository configRepository; private StandardSource source; private SourcesHandler sourceHandler; + private Supplier uuidSupplier; + @SuppressWarnings("unchecked") @BeforeEach void setUp() { configRepository = mock(ConfigRepository.class); + uuidSupplier = mock(Supplier.class); + source = generateSource(); - sourceHandler = new SourcesHandler(configRepository); + sourceHandler = new SourcesHandler(configRepository, uuidSupplier); } private StandardSource generateSource() { @@ -59,22 +69,31 @@ private StandardSource generateSource() { return new StandardSource() .withSourceId(sourceId) - .withName("presto"); + .withName("presto") + .withDocumentationUrl("https://netflix.com") + .withDockerRepository("dockerstuff") + .withDockerImageTag("12.3"); } @Test - void testListSources() throws JsonValidationException, IOException, ConfigNotFoundException { + void testListSources() throws JsonValidationException, IOException, ConfigNotFoundException, URISyntaxException { final StandardSource source2 = generateSource(); when(configRepository.listStandardSources()).thenReturn(Lists.newArrayList(source, source2)); SourceRead expectedSourceRead1 = new SourceRead() .sourceId(source.getSourceId()) - .name(source.getName()); + .name(source.getName()) + .dockerRepository(source.getDockerRepository()) + .dockerImageTag(source.getDockerImageTag()) + .documentationUrl(new URI(source.getDocumentationUrl())); SourceRead expectedSourceRead2 = new SourceRead() .sourceId(source2.getSourceId()) - .name(source2.getName()); + .name(source2.getName()) + .dockerRepository(source.getDockerRepository()) + .dockerImageTag(source.getDockerImageTag()) + .documentationUrl(new URI(source.getDocumentationUrl())); final SourceReadList actualSourceReadList = sourceHandler.listSources(); @@ -82,13 +101,16 @@ void testListSources() throws JsonValidationException, IOException, ConfigNotFou } @Test - void testGetSource() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetSource() throws JsonValidationException, ConfigNotFoundException, IOException, URISyntaxException { when(configRepository.getStandardSource(source.getSourceId())) .thenReturn(source); SourceRead expectedSourceRead = new SourceRead() .sourceId(source.getSourceId()) - .name(source.getName()); + .name(source.getName()) + .dockerRepository(source.getDockerRepository()) + .dockerImageTag(source.getDockerImageTag()) + .documentationUrl(new URI(source.getDocumentationUrl())); final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(source.getSourceId()); @@ -97,4 +119,38 @@ void testGetSource() throws JsonValidationException, ConfigNotFoundException, IO assertEquals(expectedSourceRead, actualSourceRead); } + @Test + void testCreateSource() throws URISyntaxException, ConfigNotFoundException, IOException, JsonValidationException { + final StandardSource source = generateSource(); + + when(uuidSupplier.get()).thenReturn(source.getSourceId()); + final SourceCreate create = new SourceCreate() + .name(source.getName()) + .dockerRepository(source.getDockerRepository()) + .dockerImageTag(source.getDockerImageTag()) + .documentationUrl(new URI(source.getDocumentationUrl())); + + final SourceRead expectedRead = new SourceRead() + .name(source.getName()) + .dockerRepository(source.getDockerRepository()) + .dockerImageTag(source.getDockerImageTag()) + .documentationUrl(new URI(source.getDocumentationUrl())) + .sourceId(source.getSourceId()); + + final SourceRead actualRead = sourceHandler.createSource(create); + + assertEquals(expectedRead, actualRead); + } + + @Test + void testUpdateSource() throws ConfigNotFoundException, IOException, JsonValidationException { + when(configRepository.getStandardSource(source.getSourceId())).thenReturn(source); + final String newDockerImageTag = "averydifferenttag"; + String currentTag = sourceHandler.getSource(new SourceIdRequestBody().sourceId(source.getSourceId())).getDockerImageTag(); + assertNotEquals(newDockerImageTag, currentTag); + + SourceRead sourceRead = sourceHandler.updateSource(new SourceUpdate().sourceId(source.getSourceId()).dockerImageTag(newDockerImageTag)); + assertEquals(newDockerImageTag, sourceRead.getDockerImageTag()); + } + }