diff --git a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/8442ee76-cc1d-419a-bd8b-859a090366d4.json b/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/8442ee76-cc1d-419a-bd8b-859a090366d4.json deleted file mode 100644 index 547590d7801b..000000000000 --- a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/8442ee76-cc1d-419a-bd8b-859a090366d4.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "destinationId": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6", - "destinationSpecificationId": "8442ee76-cc1d-419a-bd8b-859a090366d4", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/local-csv", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Singer CSV Target Spec", - "type": "object", - "required": ["delimiter", "quotechar"], - "additionalProperties": false, - "properties": { - "delimiter": { - "description": "Delimiter used to separate fields.", - "type": "string", - "examples": [","] - }, - "quotechar": { - "description": "The character used to quote strings containing special characters in the CSV. See python docs for more details.", - "type": "string", - "examples": ["\""] - }, - "destination_path": { - "description": "Path to the directory where csv files will be written. Check out the docs for more details on the root of this path.", - "type": "string" - } - } - } -} diff --git a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/a6655e6a-838c-4ecb-a28f-ffdcd27ec710.json b/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/a6655e6a-838c-4ecb-a28f-ffdcd27ec710.json deleted file mode 100644 index 8abd2241be5d..000000000000 --- a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/a6655e6a-838c-4ecb-a28f-ffdcd27ec710.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "destinationId": "25c5221d-dce2-4163-ade9-739ef790f503", - "destinationSpecificationId": "a6655e6a-838c-4ecb-a28f-ffdcd27ec710", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Destination Spec", - "type": "object", - "required": [ - "postgres_host", - "postgres_port", - "postgres_username", - "postgres_password", - "postgres_database", - "postgres_schema" - ], - "additionalProperties": false, - "properties": { - "postgres_host": { - "description": "Hostname of the database.", - "type": "string" - }, - "postgres_port": { - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536 - }, - "postgres_username": { - "description": "Username to use to access the database.", - "type": "string" - }, - "postgres_password": { - "description": "Password associated with the username.", - "type": "string" - }, - "postgres_database": { - "description": "Name of the database.", - "type": "string" - }, - "postgres_schema": { - "description": "Unless specifically configured, the usual value for this field is \"public\".", - "type": "string" - } - } - } -} diff --git a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/e28a1a10-214a-4051-8cf4-79b6f88719cd.json b/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/e28a1a10-214a-4051-8cf4-79b6f88719cd.json deleted file mode 100644 index c99ccde9df45..000000000000 --- a/airbyte-config/init/src/main/resources/config/DESTINATION_CONNECTION_SPECIFICATION/e28a1a10-214a-4051-8cf4-79b6f88719cd.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "destinationId": "22f6c74f-5699-40ff-833c-4a879ea40133", - "destinationSpecificationId": "e28a1a10-214a-4051-8cf4-79b6f88719cd", - "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Destination Spec", - "type": "object", - "required": ["project_id", "dataset_id", "credentials_json"], - "additionalProperties": false, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset." - }, - "dataset_id": { - "type": "string", - "description": "The BigQuery dataset id that will house replicated tables." - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key." - } - } - } -} diff --git a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/2168516a-5c9a-4582-90dc-5e3a01e3f607.json b/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/2168516a-5c9a-4582-90dc-5e3a01e3f607.json deleted file mode 100644 index 4f93a1a90d50..000000000000 --- a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/2168516a-5c9a-4582-90dc-5e3a01e3f607.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "sourceId": "decd338e-5647-4c0b-adf4-da0e75f5a750", - "sourceSpecificationId": "2168516a-5c9a-4582-90dc-5e3a01e3f607", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Source Spec", - "type": "object", - "required": ["host", "port", "user", "dbname"], - "additionalProperties": false, - "properties": { - "host": { - "description": "Hostname of the database.", - "type": "string" - }, - "port": { - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536 - }, - "user": { - "description": "Username to use to access the database.", - "type": "string" - }, - "password": { - "description": "Password associated with the username.", - "type": "string" - }, - "dbname": { - "description": "Name of the database.", - "type": "string" - } - } - } -} diff --git a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/37eb2ebf-0899-4b22-aba8-8537ec88b5a8.json b/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/37eb2ebf-0899-4b22-aba8-8537ec88b5a8.json deleted file mode 100644 index 7de00f33c055..000000000000 --- a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/37eb2ebf-0899-4b22-aba8-8537ec88b5a8.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "sourceId": "9fed261d-d107-47fd-8c8b-323023db6e20", - "sourceSpecificationId": "37eb2ebf-0899-4b22-aba8-8537ec88b5a8", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi-io", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "exchangeratesapi.io Source Spec", - "type": "object", - "required": ["start_date", "base"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start getting data from that date.", - "examples": ["YYYY-MM-DD"] - }, - "base": { - "type": "string", - "description": "ISO reference currency. See here." - } - } - } -} diff --git a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/dd42e77b-24ce-485d-8146-ee6c96d5b454.json b/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/dd42e77b-24ce-485d-8146-ee6c96d5b454.json deleted file mode 100644 index 385a1fa6ca82..000000000000 --- a/airbyte-config/init/src/main/resources/config/SOURCE_CONNECTION_SPECIFICATION/dd42e77b-24ce-485d-8146-ee6c96d5b454.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "sourceId": "e094cb9a-26de-4645-8761-65c0c425d1de", - "sourceSpecificationId": "dd42e77b-24ce-485d-8146-ee6c96d5b454", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["client_secret", "account_id", "start_date"], - "additionalProperties": false, - "properties": { - "client_secret": { - "type": "string", - "pattern": "^(s|r)k_(live|test)_[a-zA-Z0-9]+$", - "description": "Stripe API key (usually starts with 'sk_live_'; find yours here)." - }, - "account_id": { - "type": "string", - "pattern": "^acct_[a-zA-Z0-9]+$", - "description": "Your Stripe account ID (starts with 'acct_', find yours here)." - }, - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - } - } - } -} diff --git a/airbyte-integrations/src/main/java/io/airbyte/integrations/Integrations.java b/airbyte-integrations/src/main/java/io/airbyte/integrations/Integrations.java deleted file mode 100644 index 671067954427..000000000000 --- a/airbyte-integrations/src/main/java/io/airbyte/integrations/Integrations.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations; - -import java.util.UUID; - -public enum Integrations { - - POSTGRES_TAP( - UUID.fromString("2168516a-5c9a-4582-90dc-5e3a01e3f607"), - new IntegrationMapping("airbyte/integration-singer-postgres-source", "0.1.3")), - EXCHANGERATESAPI_IO_TAP( - UUID.fromString("37eb2ebf-0899-4b22-aba8-8537ec88b5a8"), - new IntegrationMapping("airbyte/integration-singer-exchangeratesapi_io-source", "0.1.3")), - STRIPE_TAP( - UUID.fromString("dd42e77b-24ce-485d-8146-ee6c96d5b454"), - new IntegrationMapping("airbyte/integration-singer-stripe-source", "0.1.2")), - POSTGRES_TARGET( - UUID.fromString("a6655e6a-838c-4ecb-a28f-ffdcd27ec710"), - new IntegrationMapping("airbyte/integration-singer-postgres-destination", "0.1.2")), - BIGQUERY_TARGET( - UUID.fromString("e28a1a10-214a-4051-8cf4-79b6f88719cd"), - new IntegrationMapping("airbyte/integration-singer-bigquery-destination", "0.1.4")), - CSV_TARGET( - UUID.fromString("8442ee76-cc1d-419a-bd8b-859a090366d4"), - new IntegrationMapping("airbyte/integration-singer-csv-destination", "0.1.1")); - - private final UUID specId; - private final IntegrationMapping integrationMapping; - - // todo (cgardens) - turn this into a map if we have enough integrations that iterating through - // the enum becomes expensive. - public static Integrations findBySpecId(UUID specId) { - for (Integrations value : values()) { - if (value.getSpecId().equals(specId)) { - return value; - } - } - throw new RuntimeException("No integrations found with spec id: " + specId); - } - - Integrations(UUID specId, IntegrationMapping integrationMapping) { - this.specId = specId; - this.integrationMapping = integrationMapping; - } - - public UUID getSpecId() { - return specId; - } - - public String getTaggedImage() { - return integrationMapping.getTaggedImage(); - } - - public static class IntegrationMapping { - - private final String image; - private final String tag; - - public IntegrationMapping(String image, String tag) { - this.image = image; - this.tag = tag; - } - - public String getTaggedImage() { - return image + ":" + tag; - } - - public String getImage() { - return image; - } - - public String getTag() { - return tag; - } - - } - -} diff --git a/airbyte-scheduler/build.gradle b/airbyte-scheduler/build.gradle index 645701bc2a0b..d6d33512c14e 100644 --- a/airbyte-scheduler/build.gradle +++ b/airbyte-scheduler/build.gradle @@ -9,7 +9,6 @@ dependencies { implementation project(':airbyte-config:models') implementation project(':airbyte-config:persistence') implementation project(':airbyte-db') - implementation project(':airbyte-integrations') implementation project(':airbyte-workers') testImplementation "org.testcontainers:postgresql:1.15.0-rc2" diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 5ee678ae06b2..dae03b9e5234 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -18,7 +18,6 @@ dependencies { implementation project(':airbyte-config:persistence') implementation project(':airbyte-config:init') implementation project(':airbyte-db') - implementation project(':airbyte-integrations') implementation project(':airbyte-scheduler') } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 2b76e851e3e1..d5505df98423 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -116,7 +116,7 @@ public ConfigurationApi(final ConfigRepository configRepository, final Scheduler webBackendConnectionsHandler = new WebBackendConnectionsHandler(connectionsHandler, sourceImplementationsHandler, jobHistoryHandler); webBackendSourceImplementationHandler = new WebBackendSourceImplementationHandler(sourceImplementationsHandler, schedulerHandler); webBackendDestinationImplementationHandler = new WebBackendDestinationImplementationHandler(destinationImplementationsHandler, schedulerHandler); - debugInfoHandler = new DebugInfoHandler(); + debugInfoHandler = new DebugInfoHandler(configRepository); } // WORKSPACE diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java index b1433afe84a8..c5d477339636 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DebugInfoHandler.java @@ -28,18 +28,25 @@ import com.google.common.collect.Lists; import io.airbyte.api.model.DebugRead; -import io.airbyte.integrations.Integrations; +import io.airbyte.commons.docker.DockerUtils; +import io.airbyte.config.persistence.ConfigRepository; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import javax.annotation.Nullable; import org.apache.commons.io.IOUtils; public class DebugInfoHandler { + private ConfigRepository configRepository; + + public DebugInfoHandler(ConfigRepository configRepository) { + this.configRepository = configRepository; + } + public DebugRead getInfo() { List> integrationImages = getIntegrationImages(); List> runningCoreImages = getRunningCoreImages(); @@ -88,10 +95,14 @@ private static List> getRunningCoreImages() { } } - private static List> getIntegrationImages() { + private List> getIntegrationImages() { try { - return Arrays.stream(Integrations.values()) - .map(Integrations::getTaggedImage) + Stream sourceImages = + configRepository.listStandardSources().stream().map(s -> DockerUtils.getTaggedImageName(s.getDockerRepository(), s.getDockerImageTag())); + Stream destinationImages = + configRepository.listStandardDestinations().stream() + .map(d -> DockerUtils.getTaggedImageName(d.getDockerRepository(), d.getDockerImageTag())); + return Stream.concat(sourceImages, destinationImages) .map(image -> { try { String hash = runAndGetOutput(Lists.newArrayList("docker", "images", "--no-trunc", "--quiet", image)); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DebugInfoHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DebugInfoHandlerTest.java index d1d04ba14884..949ae1a05dbb 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DebugInfoHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DebugInfoHandlerTest.java @@ -25,16 +25,33 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.collect.Lists; +import io.airbyte.commons.json.JsonValidationException; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.server.helpers.DestinationHelpers; +import io.airbyte.server.helpers.SourceHelpers; import java.io.IOException; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class DebugInfoHandlerTest { + private ConfigRepository configRepository; + + @BeforeEach + public void init() { + configRepository = mock(ConfigRepository.class); + } + @Test - public void testNoFailures() { - new DebugInfoHandler().getInfo(); + public void testNoFailures() throws ConfigNotFoundException, IOException, JsonValidationException { + when(configRepository.listStandardSources()).thenReturn(Lists.newArrayList(SourceHelpers.generateSource())); + when(configRepository.listStandardDestinations()).thenReturn(Lists.newArrayList(DestinationHelpers.generateDestination())); + new DebugInfoHandler(configRepository).getInfo(); } @Test