From 35761ef5a0e6090f818c6fb352714143f63a31b0 Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Sat, 23 Mar 2024 21:34:10 -0700 Subject: [PATCH] Cleanup connector generator. This commit cleans up connector generator, specifically: - Removes java sources and destinations from plopfile - Removes singer source from plopfile - Removes generic and python (non-http) source from plopfile - Cleans up documentation --- .../destination-java/.dockerignore.hbs | 3 - .../destination-java/Destination.java.hbs | 41 --- .../DestinationAcceptanceTest.java.hbs | 62 ---- .../destination-java/README.md.hbs | 71 ---- .../destination-java/build.gradle.hbs | 20 - .../destination-java/doc.md.hbs | 52 --- .../destination-java/metadata.yaml.hbs | 25 -- .../destination-java/spec.json.hbs | 21 -- .../generator/build.gradle | 5 - .../connector-templates/generator/plopfile.js | 341 +++++------------- .../source-generic/Dockerfile | 9 - .../source-generic/README.md | 45 --- .../source-generic/metadata.yaml.hbs | 23 -- .../source-java-jdbc/.dockerignore | 3 - .../source-java-jdbc/README.md | 72 ---- .../acceptance-test-config.yml.hbs | 8 - .../source-java-jdbc/build.gradle | 25 -- .../integration_tests/acceptance.py | 16 - .../source-java-jdbc/metadata.yaml.hbs | 25 -- .../{{pascalCase name}}Source.java.hbs | 52 --- .../src/main/resources/spec.json.hbs | 60 --- ...alCase name}}SourceAcceptanceTest.java.hbs | 61 ---- .../resources/dummy_config.json | 7 - .../resources/expected_spec.json.hbs | 61 ---- ...se name}}JdbcSourceAcceptanceTest.java.hbs | 75 ---- .../{{pascalCase name}}SourceTests.java.hbs | 31 -- .../README.md.hbs | 0 .../__init__.py | 0 .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 0 .../integration_tests/acceptance.py | 0 .../integration_tests/configured_catalog.json | 0 .../integration_tests/invalid_config.json | 0 .../integration_tests/sample_config.json | 0 .../integration_tests/sample_state.json | 0 .../metadata.yaml.hbs | 0 .../pyproject.toml.hbs | 0 .../secrets/config.json.hbs | 0 .../src/main.py.hbs | 0 .../source_{{snakeCase name}}/__init__.py.hbs | 0 .../manifest.yaml.hbs | 0 .../src/source_{{snakeCase name}}/run.py.hbs | 0 .../schemas/TODO.md.hbs | 0 .../schemas/customers.json | 0 .../schemas/employees.json | 0 .../source_{{snakeCase name}}/source.py.hbs | 0 .../source-python-http-api/README.md.hbs | 105 ------ .../integration_tests/abnormal_state.json | 5 - .../integration_tests/acceptance.py | 16 - .../integration_tests/configured_catalog.json | 22 -- .../integration_tests/invalid_config.json | 3 - .../integration_tests/sample_config.json | 3 - .../integration_tests/sample_state.json | 5 - .../source-python-http-api/metadata.yaml.hbs | 34 -- .../source-python-http-api/pyproject.toml.hbs | 27 -- .../secrets/config.json.hbs | 3 - .../source-python-http-api/src/main.py.hbs | 8 - .../source_{{snakeCase name}}/__init__.py.hbs | 8 - .../src/source_{{snakeCase name}}/run.py.hbs | 13 - .../source_{{snakeCase name}}/source.py.hbs | 206 ----------- .../source_{{snakeCase name}}/spec.yaml.hbs | 12 - .../unit_tests/__init__.py | 3 - .../source-python/README.md.hbs | 3 + .../integration_tests/acceptance.py | 2 +- .../integration_tests/configured_catalog.json | 15 +- .../source-python/metadata.yaml.hbs | 8 +- .../source-python/secrets/config.json.hbs | 2 +- .../source_{{snakeCase name}}/schemas/TODO.md | 0 .../schemas/customers.json | 0 .../schemas/employees.json | 0 .../source_{{snakeCase name}}/source.py.hbs | 301 +++++++++++----- .../source_{{snakeCase name}}/spec.yaml.hbs | 5 +- .../unit_tests}/__init__.py | 0 .../test_incremental_streams.py.hbs | 0 .../unit_tests/test_source.py.hbs | 0 .../unit_tests/test_streams.py.hbs | 0 .../source-python/unit_tests/unit_test.py.hbs | 7 - docs/connector-development/README.md | 1 - .../config-based/low-code-cdk-overview.md | 1 - .../tutorials/cdk-speedrun.md | 5 +- .../creating-the-source.md | 2 +- .../getting-started.md | 7 +- .../resources/developing-locally.md | 2 +- 83 files changed, 321 insertions(+), 1727 deletions(-) delete mode 100644 airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/Destination.java.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/README.md.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/build.gradle.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/doc.md.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs delete mode 100644 airbyte-integrations/connector-templates/destination-java/spec.json.hbs delete mode 100644 airbyte-integrations/connector-templates/source-generic/Dockerfile delete mode 100644 airbyte-integrations/connector-templates/source-generic/README.md delete mode 100644 airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/README.md delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-config.yml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/build.gradle delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/integration_tests/acceptance.py delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/dummy_config.json delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/expected_spec.json.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs delete mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/README.md.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/__init__.py (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/__init__.py (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/abnormal_state.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/acceptance.py (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/configured_catalog.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/invalid_config.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/sample_config.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/integration_tests/sample_state.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/metadata.yaml.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/pyproject.toml.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/secrets/config.json.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/main.py.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/__init__.py.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/manifest.yaml.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/run.py.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/schemas/TODO.md.hbs (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/schemas/customers.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/schemas/employees.json (100%) rename airbyte-integrations/connector-templates/{source-configuration-based => source-low-code}/src/source_{{snakeCase name}}/source.py.hbs (100%) delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/abnormal_state.json delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/acceptance.py delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/configured_catalog.json delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/invalid_config.json delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_config.json delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_state.json delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/pyproject.toml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/secrets/config.json.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/src/main.py.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/__init__.py.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/run.py.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/source.py.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/spec.yaml.hbs delete mode 100644 airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/src/source_{{snakeCase name}}/schemas/TODO.md (100%) rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/src/source_{{snakeCase name}}/schemas/customers.json (100%) rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/src/source_{{snakeCase name}}/schemas/employees.json (100%) rename airbyte-integrations/connector-templates/{source-python-http-api/integration_tests => source-python/unit_tests}/__init__.py (100%) rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/unit_tests/test_incremental_streams.py.hbs (100%) rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/unit_tests/test_source.py.hbs (100%) rename airbyte-integrations/connector-templates/{source-python-http-api => source-python}/unit_tests/test_streams.py.hbs (100%) delete mode 100644 airbyte-integrations/connector-templates/source-python/unit_tests/unit_test.py.hbs diff --git a/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs b/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs b/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs deleted file mode 100644 index f5785a3be592..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.BaseConnector; -import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; -import io.airbyte.cdk.integrations.base.Destination; -import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.protocol.models.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class {{properCase name}}Destination extends BaseConnector implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger({{properCase name}}Destination.class); - - public static void main(String[] args) throws Exception { - new IntegrationRunner(new {{properCase name}}Destination()).run(args); - } - - @Override - public AirbyteConnectionStatus check(JsonNode config) throws Exception { - // TODO - return null; - } - - @Override - public AirbyteMessageConsumer getConsumer(JsonNode config, - ConfiguredAirbyteCatalog configuredCatalog, - Consumer outputRecordCollector) throws Exception{ - // TODO - return null; - } - -} diff --git a/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs deleted file mode 100644 index 1663f1066429..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; -import java.io.IOException; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class {{properCase name}}DestinationAcceptanceTest extends DestinationAcceptanceTest { - - private static final Logger LOGGER = LoggerFactory.getLogger({{properCase name}}DestinationAcceptanceTest.class); - - private JsonNode configJson; - - @Override - protected String getImageName() { - return "airbyte/destination-{{snakeCase name}}:dev"; - } - - @Override - protected JsonNode getConfig() { - // TODO: Generate the configuration JSON file to be used for running the destination during the test - // configJson can either be static and read from secrets/config.json directly - // or created in the setup method - return configJson; - } - - @Override - protected JsonNode getFailCheckConfig() { - // TODO return an invalid config which, when used to run the connector's check connection operation, - // should result in a failed connection check - return null; - } - - @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) - throws IOException { - // TODO Implement this method to retrieve records which written to the destination by the connector. - // Records returned from this method will be compared against records provided to the connector - // to verify they were written correctly - return null; - } - - @Override - protected void setup(TestDestinationEnv testEnv) { - // TODO Implement this method to run any setup actions needed before every test case - } - - @Override - protected void tearDown(TestDestinationEnv testEnv) { - // TODO Implement this method to run any cleanup actions needed after every test case - } - -} diff --git a/airbyte-integrations/connector-templates/destination-java/README.md.hbs b/airbyte-integrations/connector-templates/destination-java/README.md.hbs deleted file mode 100644 index 390002dca65d..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/README.md.hbs +++ /dev/null @@ -1,71 +0,0 @@ -# Destination {{capitalCase name}} - -This is the repository for the {{capitalCase name}} destination connector in Java. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.com/integrations/destinations/{{dashCase name}}). - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:build -``` - -#### Create credentials -**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. -Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. - -**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.com/connector-development#using-credentials-in-ci) to set up the credentials. - -### Locally running the connector docker image - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:buildConnectorImage -``` -Once built, the docker image name and tag will be `airbyte/source-{{dashCase name}}:dev`. - - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/destination-{{dashCase name}}:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-{{dashCase name}}:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-{{dashCase name}}:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-{{dashCase name}}:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -## Testing -We use `JUnit` for Java tests. - -### Unit and Integration Tests -Place unit tests under `src/test/io/airbyte/integrations/destinations/{{snakeCase name}}`. - -#### Acceptance Tests -Airbyte has a standard test suite that all destination connectors must pass. Implement the `TODO`s in -`src/test-integration/java/io/airbyte/integrations/destinations/{{snakeCase name}}DestinationAcceptanceTest.java`. - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:check -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:integrationTest -``` - -## Dependency Management - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-{{dashCase name}} test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/{{dashCase name}}.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs deleted file mode 100644 index a999413ffe2a..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs +++ /dev/null @@ -1,20 +0,0 @@ -plugins { - id 'application' - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-destinations'] - useLocalCdk = true -} - -airbyteJavaConnector.addCdkDependencies() - -application { - mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination' -} - -dependencies { - implementation libs.airbyte.protocol -} diff --git a/airbyte-integrations/connector-templates/destination-java/doc.md.hbs b/airbyte-integrations/connector-templates/destination-java/doc.md.hbs deleted file mode 100644 index 85a8697b44a6..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/doc.md.hbs +++ /dev/null @@ -1,52 +0,0 @@ -# {{capitalCase name}} - -TODO: update this doc - -## Sync overview - -### Output schema - -Is the output schema fixed (e.g: for an API like Stripe)? If so, point to the connector's schema (e.g: link to Stripe’s documentation) or describe the schema here directly (e.g: include a diagram or paragraphs describing the schema). - -Describe how the connector's schema is mapped to Airbyte concepts. An example description might be: "MagicDB tables become Airbyte Streams and MagicDB columns become Airbyte Fields. In addition, an extracted\_at column is appended to each row being read." - -### Data type mapping - -This section should contain a table mapping each of the connector's data types to Airbyte types. At the moment, Airbyte uses the same types used by [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html). `string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number` are the most commonly used data types. - -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | - - -### Features - -This section should contain a table with the following format: - -| Feature | Supported?(Yes/No) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | | | -| Incremental Sync | | | -| Replicate Incremental Deletes | | | -| For databases, WAL/Logical replication | | | -| SSL connection | | | -| SSH Tunnel Support | | | -| (Any other source-specific features) | | | - -### Performance considerations - -Could this connector hurt the user's database/API/etc... or put too much strain on it in certain circumstances? For example, if there are a lot of tables or rows in a table? What is the breaking point (e.g: 100mm> records)? What can the user do to prevent this? (e.g: use a read-only replica, or schedule frequent syncs, etc..) - -## Getting started - -### Requirements - -* What versions of this connector does this implementation support? (e.g: `postgres v3.14 and above`) -* What configurations, if any, are required on the connector? (e.g: `buffer_size > 1024`) -* Network accessibility requirements -* Credentials/authentication requirements? (e.g: A DB user with read permissions on certain tables) - -### Setup guide - -For each of the above high-level requirements as appropriate, add or point to a follow-along guide. See existing source or destination guides for an example. - -For each major cloud provider we support, also add a follow-along guide for setting up Airbyte to connect to that destination. See the Postgres destination guide for an example of what this should look like. diff --git a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs deleted file mode 100644 index 5f67617a6fe2..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs +++ /dev/null @@ -1,25 +0,0 @@ -data: - allowedHosts: - hosts: - - TODO # Please change to the hostname of the source. - registries: - oss: - enabled: true - cloud: - enabled: false - connectorSubtype: database - connectorType: destination - definitionId: {{generateDefinitionId}} - dockerImageTag: 0.1.0 - dockerRepository: airbyte/destination-{{dashCase name}} - githubIssueLabel: destination-{{dashCase name}} - icon: {{dashCase name}}.svg - license: MIT - name: {{capitalCase name}} - releaseDate: TODO - releaseStage: alpha - supportLevel: community - documentationUrl: https://docs.airbyte.com/integrations/destinations/{{dashCase name}} - tags: - - language:python -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/destination-java/spec.json.hbs b/airbyte-integrations/connector-templates/destination-java/spec.json.hbs deleted file mode 100644 index b56ea8a843fc..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/spec.json.hbs +++ /dev/null @@ -1,21 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/{{dashCase name}}", - "supportsIncremental": TODO, - "supported_destination_sync_modes": ["TODO"], - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "TODO", - "type": "object", - "required": [ - "TODO" - ], - "properties": { - "TODO_sample_field": { - "title": "Sample Field", - "type": "string", - "description": "", - "examples": [""] - } - } - } -} diff --git a/airbyte-integrations/connector-templates/generator/build.gradle b/airbyte-integrations/connector-templates/generator/build.gradle index 2a93a2f67a08..aef81870c054 100644 --- a/airbyte-integrations/connector-templates/generator/build.gradle +++ b/airbyte-integrations/connector-templates/generator/build.gradle @@ -26,10 +26,5 @@ def addScaffoldTemplateTask(name, packageName, outputDirName, scaffoldParams=[]) generateScaffolds.configure { dependsOn task } } - addScaffoldTemplateTask('Python Source', 'scaffold-source-python', 'source-scaffold-source-python') -addScaffoldTemplateTask('Python HTTP API Source', 'scaffold-source-http', 'source-scaffold-source-http') -addScaffoldTemplateTask('Java JDBC Source', 'scaffold-java-jdbc', 'source-scaffold-java-jdbc') addScaffoldTemplateTask('Python Destination', 'scaffold-destination-python', 'destination-scaffold-destination-python') -// TODO: enable Singer template testing -//addScaffoldTask('source-python-singer', ['tap-exchangeratesapi']) diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index f1a97cc3f942..055afa300f7f 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -1,15 +1,21 @@ -'use strict'; -const path = require('path'); -const uuid = require('uuid'); -const capitalCase = require('capital-case'); -const changeCase = require('change-case') -const getSuccessMessage = function(connectorName, outputPath, additionalMessage){ - return ` +"use strict"; +const path = require("path"); +const uuid = require("uuid"); +const capitalCase = require("capital-case"); +const changeCase = require("change-case"); +const getSuccessMessage = function ( + connectorName, + outputPath, + additionalMessage +) { + return ` πŸš€ πŸš€ πŸš€ πŸš€ πŸš€ πŸš€ Success! -Your ${connectorName} connector has been created at .${path.resolve(outputPath)}. +Your ${connectorName} connector has been created at .${path.resolve( + outputPath +)}. Follow the TODOs in the generated module to implement your connector. @@ -19,129 +25,137 @@ https://discuss.airbyte.io/c/connector-development/16 We're always happy to provide any support! ${additionalMessage || ""} -` -} +`; +}; module.exports = function (plop) { - const docRoot = '../../../docs/integrations'; + const connectorAcceptanceTestFilesInputRoot = + "../connector_acceptance_test_files"; - const connectorAcceptanceTestFilesInputRoot = '../connector_acceptance_test_files'; + const pythonSourceInputRoot = "../source-python"; + const lowCodeSourceInputRoot = "../source-low-code"; + const pythonDestinationInputRoot = "../destination-python"; - const pythonSourceInputRoot = '../source-python'; - const singerSourceInputRoot = '../source-singer'; - const genericSourceInputRoot = '../source-generic'; - const genericJdbcSourceInputRoot = '../source-java-jdbc'; - const httpApiInputRoot = '../source-python-http-api'; - const lowCodeSourceInputRoot = '../source-configuration-based'; - const javaDestinationInput = '../destination-java'; - const pythonDestinationInputRoot = '../destination-python'; + const outputDir = "../../connectors"; - const outputDir = '../../connectors'; const pythonSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`; - const singerSourceOutputRoot = `${outputDir}/source-{{dashCase name}}-singer`; - const genericSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`; - const genericJdbcSourceOutputRoot = `${outputDir}/source-{{dashCase name}}`; - const httpApiOutputRoot = `${outputDir}/source-{{dashCase name}}`; - const javaDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; const pythonDestinationOutputRoot = `${outputDir}/destination-{{dashCase name}}`; - const sourceConnectorImagePrefix = 'airbyte/source-' - const sourceConnectorImageTag = 'dev' - const defaultSpecPathFolderPrefix = 'source_' - const specFileName = 'spec.yaml' + const sourceConnectorImagePrefix = "airbyte/source-"; + const sourceConnectorImageTag = "dev"; + const defaultSpecPathFolderPrefix = "source_"; + + const specFileName = "spec.yaml"; - plop.setHelper('capitalCase', function(name) { + plop.setHelper("capitalCase", function (name) { return capitalCase.capitalCase(name); }); - plop.setHelper('generateDefinitionId', function() { + plop.setHelper("generateDefinitionId", function () { // if the env var CI is set then return a fixed FAKE uuid so that the tests are deterministic if (process.env.CI) { - return 'FAKE-UUID-0000-0000-000000000000'; + return "FAKE-UUID-0000-0000-000000000000"; } return uuid.v4().toLowerCase(); }); - plop.setHelper('connectorImage', function() { - let suffix = "" - if (typeof this.connectorImageNameSuffix !== 'undefined') { - suffix = this.connectorImageNameSuffix + plop.setHelper("connectorImage", function () { + let suffix = ""; + if (typeof this.connectorImageNameSuffix !== "undefined") { + suffix = this.connectorImageNameSuffix; } - return `${sourceConnectorImagePrefix}${changeCase.paramCase(this.name)}${suffix}:${sourceConnectorImageTag}` + return `${sourceConnectorImagePrefix}${changeCase.paramCase(this.name)}${suffix}:${sourceConnectorImageTag}`; }); - plop.setHelper('specPath', function() { - let suffix = "" - if (typeof this.specPathFolderSuffix !== 'undefined') { - suffix = this.specPathFolderSuffix + plop.setHelper("specPath", function () { + let suffix = ""; + if (typeof this.specPathFolderSuffix !== "undefined") { + suffix = this.specPathFolderSuffix; } - let inSubFolder = true - if (typeof this.inSubFolder !== 'undefined') { - inSubFolder = this.inSubFolder + let inSubFolder = true; + if (typeof this.inSubFolder !== "undefined") { + inSubFolder = this.inSubFolder; } if (inSubFolder) { - return `${defaultSpecPathFolderPrefix}${changeCase.snakeCase(this.name)}${suffix}/${specFileName}` + return `${defaultSpecPathFolderPrefix}${changeCase.snakeCase( + this.name + )}${suffix}/${specFileName}`; } else { - return specFileName + return specFileName; } }); - - plop.setActionType('emitSuccess', function(answers, config, plopApi){ - console.log(getSuccessMessage(answers.name, plopApi.renderString(config.outputPath, answers), config.message)); + plop.setActionType("emitSuccess", function (answers, config, plopApi) { + console.log( + getSuccessMessage( + answers.name, + plopApi.renderString(config.outputPath, answers), + config.message + ) + ); }); - plop.setGenerator('Python Destination', { - description: 'Generate a destination connector written in Python', + plop.setGenerator("Python CDK Destination", { + description: "Generate a destination connector based on Python CDK.", prompts: [ - {type:'input', name:'name', 'message': 'Connector name e.g: redis'}, + { type: "input", name: "name", message: "Connector name e.g: redis" }, ], actions: [ { abortOnFail: true, - type:'addMany', + type: "addMany", destination: pythonDestinationOutputRoot, base: pythonDestinationInputRoot, templateFiles: `${pythonDestinationInputRoot}/**/**`, }, - {type: 'emitSuccess', outputPath: pythonDestinationOutputRoot} - ] - }) + { type: "emitSuccess", outputPath: pythonDestinationOutputRoot }, + ], + }); - plop.setGenerator('Python HTTP API Source', { - description: 'Generate a Source that pulls data from a synchronous HTTP API.', + plop.setGenerator("Python CDK Source", { + description: + "Generate a source connector based on Python CDK.", prompts: [ - {type: 'input', name: 'name', message: 'Source name e.g: "google-analytics"'}, + { + type: "input", + name: "name", + message: 'Source name e.g: "google-analytics"', + }, ], actions: [ { abortOnFail: true, - type:'addMany', - destination: httpApiOutputRoot, - base: httpApiInputRoot, - templateFiles: `${httpApiInputRoot}/**/**`, + type: "addMany", + destination: pythonSourceOutputRoot, + base: pythonSourceInputRoot, + templateFiles: `${pythonSourceInputRoot}/**/**`, }, // common acceptance tests { abortOnFail: true, - type:'addMany', - destination: httpApiOutputRoot, + type: "addMany", + destination: pythonSourceOutputRoot, base: connectorAcceptanceTestFilesInputRoot, templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, }, - {type: 'emitSuccess', outputPath: httpApiOutputRoot} - ] + { type: "emitSuccess", outputPath: pythonSourceOutputRoot }, + ], }); - plop.setGenerator('Configuration Based Source', { - description: 'Generate a Source that is described using a low code configuration file', + plop.setGenerator("Low-code Source", { + description: + "Generate a source based on the low-code CDK.", prompts: [ - {type: 'input', name: 'name', message: 'Source name e.g: "google-analytics"'}, + { + type: "input", + name: "name", + message: 'Source name e.g: "google-analytics"', + }, ], - actions: [ + actions: [ { abortOnFail: true, - type:'addMany', + type: "addMany", destination: pythonSourceOutputRoot, base: lowCodeSourceInputRoot, templateFiles: `${lowCodeSourceInputRoot}/**/**`, @@ -149,189 +163,12 @@ module.exports = function (plop) { // common acceptance tests { abortOnFail: true, - type:'addMany', + type: "addMany", destination: pythonSourceOutputRoot, base: connectorAcceptanceTestFilesInputRoot, templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, }, - {type: 'emitSuccess', outputPath: pythonSourceOutputRoot} - ] - }); - - plop.setGenerator('Python Singer Source', { - description: 'Generate a Singer-tap-based Airbyte Source.', - prompts: [ - {type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"', filter: function (name) { - return name.endsWith('-singer') ? name.replace(/-singer$/, '') : name; - }}, - {type: 'input', name: 'tap_name', message: 'Singer tap package e.g "tap-mixpanel"'}, - ], - actions: [ - { - abortOnFail: true, - type:'addMany', - destination: singerSourceOutputRoot, - base: singerSourceInputRoot, - templateFiles: `${singerSourceInputRoot}/**/**`, - }, - // common acceptance tests - { - abortOnFail: true, - type:'addMany', - destination: singerSourceOutputRoot, - base: connectorAcceptanceTestFilesInputRoot, - templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, - data: { - connectorImageNameSuffix: "-singer", - specPathFolderSuffix: "_singer" - } - }, - { - type:'add', - abortOnFail: true, - templateFile: `${singerSourceInputRoot}/.gitignore.hbs`, - path: `${singerSourceOutputRoot}/.gitignore` - }, - { - type:'add', - abortOnFail: true, - templateFile: `${singerSourceInputRoot}/.dockerignore.hbs`, - path: `${singerSourceOutputRoot}/.dockerignore` - }, - {type: 'emitSuccess', outputPath: singerSourceOutputRoot}, - ] - }); - - plop.setGenerator('Python Source', { - description: 'Generate a minimal Python Airbyte Source Connector that works with any kind of data source. Use this if none of the other Python templates serve your use case.', - prompts: [ - {type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}, - ], - actions: [ - { - abortOnFail: true, - type:'addMany', - destination: pythonSourceOutputRoot, - base: pythonSourceInputRoot, - templateFiles: `${pythonSourceInputRoot}/**/**`, - }, - // common acceptance tests - { - abortOnFail: true, - type:'addMany', - destination: pythonSourceOutputRoot, - base: connectorAcceptanceTestFilesInputRoot, - templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, - }, - {type: 'emitSuccess', outputPath: pythonSourceOutputRoot, message: "For a checklist of what to do next go to https://docs.airbyte.com/connector-development/tutorials/building-a-python-source"}] - }); - - plop.setGenerator('Java JDBC Source', { - description: 'Generate a minimal Java JDBC Airbyte Source Connector.', - prompts: [ - {type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "mysql"'}, - ], - actions: [ - { - abortOnFail: true, - type:'addMany', - destination: genericJdbcSourceOutputRoot, - base: genericJdbcSourceInputRoot, - templateFiles: `${genericJdbcSourceInputRoot}/**/**`, - }, - {type: 'emitSuccess', outputPath: genericJdbcSourceOutputRoot} - ] - }); - - plop.setGenerator('Generic Source', { - description: 'Use if none of the other templates apply to your use case.', - prompts: [ - {type: 'input', name: 'name', message: 'Source name, without the "source-" prefix e.g: "google-analytics"'}, - ], - actions: [ - { - abortOnFail: true, - type:'addMany', - destination: genericSourceOutputRoot, - base: genericSourceInputRoot, - templateFiles: `${genericSourceInputRoot}/**/**`, - }, - // common acceptance tests - { - abortOnFail: true, - type:'addMany', - destination: genericSourceOutputRoot, - base: connectorAcceptanceTestFilesInputRoot, - templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, - data: { - inSubFolder: false - } - }, - {type: 'emitSuccess', outputPath: genericSourceOutputRoot} - ] - }); - - plop.setGenerator('Java Destination', { - description: 'Generate a Java Destination Connector.', - prompts: [ - { - type: 'input', - name: 'name', - message: 'Destination name, without the "destination-" prefix e.g: "google-pubsub"', - }, + { type: "emitSuccess", outputPath: pythonSourceOutputRoot }, ], - actions: [ - // Gradle - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/build.gradle.hbs`, - path: `${javaDestinationOutputRoot}/build.gradle` - }, - // Docker - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/.dockerignore.hbs`, - path: `${javaDestinationOutputRoot}/.dockerignore` - }, - // Java - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/Destination.java.hbs`, - path: `${javaDestinationOutputRoot}/src/main/java/io/airbyte/integrations/destination/{{snakeCase name}}/{{properCase name}}Destination.java` - }, - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/DestinationAcceptanceTest.java.hbs`, - path: `${javaDestinationOutputRoot}/src/test-integration/java/io/airbyte/integrations/destination/{{snakeCase name}}/{{properCase name}}DestinationAcceptanceTest.java` - }, - // Doc - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/README.md.hbs`, - path: `${javaDestinationOutputRoot}/README.md` - }, - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/doc.md.hbs`, - path: `${docRoot}/destinations/{{dashCase name}}.md` - }, - // Definition - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/spec.json.hbs`, - path: `${javaDestinationOutputRoot}/src/main/resources/spec.json` - }, - { - type: 'emitSuccess', - outputPath: javaDestinationOutputRoot, - } - ] }); }; diff --git a/airbyte-integrations/connector-templates/source-generic/Dockerfile b/airbyte-integrations/connector-templates/source-generic/Dockerfile deleted file mode 100644 index 846e34ccc961..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM scratch - -## TODO Add your dockerfile instructions here -## TODO uncomment the below line. This is required for Kubernetes compatibility. -# ENV AIRBYTE_ENTRYPOINT="update this with the command you use for an entrypoint" - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.name=airbyte/source-{{dashCase name}} -LABEL io.airbyte.version=0.1.0 diff --git a/airbyte-integrations/connector-templates/source-generic/README.md b/airbyte-integrations/connector-templates/source-generic/README.md deleted file mode 100644 index 80f439a81ec5..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# {{capitalCase name}} Source - -This is the repository for the {{capitalCase name}} source connector. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}). - -## Local development - -### Prerequisites -* If you are using Python for connector development, minimal required version `= 3.7.0` -* Valid credentials (see the "Create credentials section for instructions) -TODO: _which languages and tools does a user need to develop on this connector? add them to the bullet list above_ - -### Iteration -TODO: _which commands should a developer use to run this connector locally?_ - -### Testing -#### Unit Tests -TODO: _how can a user run unit tests?_ - -#### Integration Tests -TODO: _how can a user run integration tests?_ -_this section is currently under construction -- please reach out to us on Slack for help with setting up Airbyte's standard test suite_ - - -### Locally running the connector docker image - -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/{{dashCase name}}:dev -``` - -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-{{dashCase name}}:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-{{dashCase name}}:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `spec.json` file. `secrets` is gitignored by default. - -**If you are an Airbyte core member**, copy the credentials from Lastpass under the secret name `source {{dashCase name}} test creds` -and place them into `secrets/config.json`. diff --git a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs deleted file mode 100644 index 59f8bcf9b580..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs +++ /dev/null @@ -1,23 +0,0 @@ -data: - allowedHosts: - hosts: - - TODO # Please change to the hostname of the source. - registries: - oss: - enabled: true - cloud: - enabled: false - connectorSubtype: api - connectorType: source - definitionId: {{generateDefinitionId}} - dockerImageTag: 0.1.0 - dockerRepository: airbyte/source-{{dashCase name}} - githubIssueLabel: source-{{dashCase name}} - icon: {{dashCase name}}.svg - license: MIT - name: {{capitalCase name}} - releaseDate: TODO - releaseStage: alpha - supportLevel: community - documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore b/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore deleted file mode 100644 index e4fbece78752..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build/distributions diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/README.md b/airbyte-integrations/connector-templates/source-java-jdbc/README.md deleted file mode 100644 index 47e23b91d261..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# Source {{capitalCase name}} - -This is the repository for the {{capitalCase name}} source connector in Java. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}). - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build -``` - -#### Create credentials -**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. -Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. - -**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.com/connector-development#using-credentials-in-ci) to set up the credentials. - -### Locally running the connector docker image - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:buildConnectorImage -``` - -Once built, the docker image name and tag will be `airbyte/source-{{dashCase name}}:dev`. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-{{dashCase name}}:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-{{dashCase name}}:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -## Testing -We use `JUnit` for Java tests. - -### Unit and Integration Tests -Place unit tests under `src/test/...` -Place integration tests in `src/test-integration/...` - -#### Acceptance Tests -Airbyte has a standard test suite that all source connectors must pass. Implement the `TODO`s in -`src/test-integration/java/io/airbyte/integrations/sources/{{snakeCase name}}SourceAcceptanceTest.java`. - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:integrationTest -``` - -## Dependency Management - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` -2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). -3. Make sure the `metadata.yaml` content is up to date. -4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-config.yml.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-config.yml.hbs deleted file mode 100644 index 314351c4290e..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-config.yml.hbs +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-{{dashCase name}}:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle deleted file mode 100644 index c1991bdadd16..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id 'application' - id 'airbyte-java-connector' -} - -airbyteJavaConnector { - cdkVersionRequired = '0.2.0' - features = ['db-sources'] - useLocalCdk = true -} - -airbyteJavaConnector.addCdkDependencies() - -application { - mainClass = 'io.airbyte.integrations.source.{{dashCase name}}.{{pascalCase name}}Source' -} - -dependencies { - - //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - - testImplementation 'org.apache.commons:commons-lang3:3.11' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-{{dashCase name}}') -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-java-jdbc/integration_tests/acceptance.py deleted file mode 100644 index 9e6409236281..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/integration_tests/acceptance.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("connector_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs deleted file mode 100644 index 5e325ba27e88..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs +++ /dev/null @@ -1,25 +0,0 @@ -data: - allowedHosts: - hosts: - - TODO # Please change to the hostname of the source. - registries: - oss: - enabled: true - cloud: - enabled: false - connectorSubtype: database - connectorType: source - definitionId: {{generateDefinitionId}} - dockerImageTag: 0.1.0 - dockerRepository: airbyte/source-{{dashCase name}} - githubIssueLabel: source-{{dashCase name}} - icon: {{dashCase name}}.svg - license: MIT - name: {{capitalCase name}} - releaseDate: TODO - supportLevel: community - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs deleted file mode 100644 index a6b204586439..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.db.jdbc.JdbcUtils; -import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.cdk.integrations.base.IntegrationRunner; -import io.airbyte.cdk.integrations.base.Source; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; -import java.sql.JDBCType; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class {{pascalCase name}}Source extends AbstractJdbcSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}Source.class); - - // TODO insert your driver name. Ex: "com.microsoft.sqlserver.jdbc.SQLServerDriver" - static final String DRIVER_CLASS = "driver_name_here"; - - public {{pascalCase name}}Source() { - // TODO: if the JDBC driver does not support custom fetch size, use NoOpStreamingQueryConfig - // instead of AdaptiveStreamingQueryConfig. - super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); - } - - // TODO The config is based on spec.json, update according to your DB - @Override - public JsonNode toDatabaseConfig(final JsonNode config) { - // TODO create DB config. Ex: "Jsons.jsonNode(ImmutableMap.builder().put("username", - // userName).put("password", pas)...build()); - return null; - } - - @Override - public Set getExcludedInternalNameSpaces() { - // TODO Add tables to exclude, Ex "INFORMATION_SCHEMA", "sys", "spt_fallback_db", etc - return Set.of(""); - } - - public static void main(final String[] args) throws Exception { - final Source source = new {{pascalCase name}}Source(); - LOGGER.info("starting source: {}", {{pascalCase name}}Source.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", {{pascalCase name}}Source.class); - } - -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs deleted file mode 100644 index c839b40a716b..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs +++ /dev/null @@ -1,60 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/{{snakeCase name}}", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "{{pascalCase name}} Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "replication_method"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "Username", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "title": "JDBC URL params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", - "type": "string", - "order": 5 - }, - "replication_method": { - "title": "Replication method", - "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "type": "string", - "order": 6, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] - } - } - } -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs deleted file mode 100644 index eba3f8c53e74..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.HashMap; - -public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTest { - - private JsonNode config; - - @Override - protected void setupEnvironment(final TestDestinationEnv testEnv) { - // TODO create new container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g");" - // TODO make container started. Ex: "container.start();" - // TODO init JsonNode config - // TODO crete airbyte Database object "Databases.createJdbcDatabase(...)" - // TODO insert test data to DB. Ex: "database.execute(connection-> ...)" - // TODO close Database. Ex: "database.close();" - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - // TODO close container that was initialized in setup() method. Ex: "container.close();" - } - - @Override - protected String getImageName() { - return "airbyte/source-{{dashCase name}}:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - // TODO Return the ConfiguredAirbyteCatalog with ConfiguredAirbyteStream objects - return null; - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/dummy_config.json b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/dummy_config.json deleted file mode 100644 index 483d12bc3cd1..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/dummy_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "host": "default", - "port": 5555, - "database": "default", - "username": "default", - "replication_method": "STANDARD" -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/expected_spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/expected_spec.json.hbs deleted file mode 100644 index 5d3ba7fc2413..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/resources/expected_spec.json.hbs +++ /dev/null @@ -1,61 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/{{snakeCase name}}", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "{{pascalCase name}} Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "replication_method"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "title": "Database", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "Username", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "title": "JDBC URL params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", - "type": "string", - "order": 5 - }, - "replication_method": { - "title": "Replication method", - "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "type": "string", - "order": 6, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs deleted file mode 100644 index c2046c3a49da..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import java.sql.JDBCType; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class {{pascalCase name}}JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}JdbcSourceAcceptanceTest.class); - - // TODO declare a test container for DB. EX: org.testcontainers.containers.OracleContainer - - @BeforeAll - static void init() { - // Oracle returns uppercase values - // TODO init test container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g")" - // TODO start container. Ex: "container.start();" - } - - @BeforeEach - public void setup() throws Exception { - // TODO init config. Ex: "config = Jsons.jsonNode(ImmutableMap.builder().put("host", - // host).put("port", port)....build()); - super.setup(); - } - - @AfterEach - public void tearDown() { - // TODO clean used resources - } - - @Override - public AbstractJdbcSource getSource() { - return new {{pascalCase name}}Source(); - } - - @Override - public boolean supportsSchemas() { - // TODO check if your db supports it and update method accordingly - return false; - } - - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return {{pascalCase name}}Source.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - // TODO - return null; - } - - @AfterAll - static void cleanUp() { - // TODO close the container. Ex: "container.close();" - } - -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs deleted file mode 100644 index 082504f2e4b8..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.{{snakeCase name}}; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.cdk.db.Database; -import org.junit.jupiter.api.Test; - -public class {{pascalCase name}}SourceTests { - - private JsonNode config; - private Database database; - - @Test - public void testSettingTimezones() throws Exception { - // TODO init your container. Ex: "new - // org.testcontainers.containers.MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense();" - // TODO start the container. Ex: "container.start();" - // TODO prepare DB config. Ex: "config = getConfig(container, dbName, - // "serverTimezone=Europe/London");" - // TODO create DB, grant all privileges, etc. - // TODO check connection status. Ex: "AirbyteConnectionStatus check = new - // ScaffoldJavaJdbcGenericSource().check(config);" - // TODO assert connection status. Ex: "assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, - // check.getStatus());" - // TODO cleanup used resources and close used container. Ex: "container.close();" - } - -} diff --git a/airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs b/airbyte-integrations/connector-templates/source-low-code/README.md.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs rename to airbyte-integrations/connector-templates/source-low-code/README.md.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/__init__.py b/airbyte-integrations/connector-templates/source-low-code/__init__.py similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/__init__.py rename to airbyte-integrations/connector-templates/source-low-code/__init__.py diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/__init__.py b/airbyte-integrations/connector-templates/source-low-code/integration_tests/__init__.py similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/__init__.py rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/__init__.py diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/abnormal_state.json b/airbyte-integrations/connector-templates/source-low-code/integration_tests/abnormal_state.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/abnormal_state.json rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/abnormal_state.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/acceptance.py rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/acceptance.py diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/configured_catalog.json b/airbyte-integrations/connector-templates/source-low-code/integration_tests/configured_catalog.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/configured_catalog.json rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/invalid_config.json b/airbyte-integrations/connector-templates/source-low-code/integration_tests/invalid_config.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/invalid_config.json rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/invalid_config.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/sample_config.json b/airbyte-integrations/connector-templates/source-low-code/integration_tests/sample_config.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/sample_config.json rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/sample_config.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/integration_tests/sample_state.json b/airbyte-integrations/connector-templates/source-low-code/integration_tests/sample_state.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/integration_tests/sample_state.json rename to airbyte-integrations/connector-templates/source-low-code/integration_tests/sample_state.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-low-code/metadata.yaml.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs rename to airbyte-integrations/connector-templates/source-low-code/metadata.yaml.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/pyproject.toml.hbs b/airbyte-integrations/connector-templates/source-low-code/pyproject.toml.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/pyproject.toml.hbs rename to airbyte-integrations/connector-templates/source-low-code/pyproject.toml.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/secrets/config.json.hbs b/airbyte-integrations/connector-templates/source-low-code/secrets/config.json.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/secrets/config.json.hbs rename to airbyte-integrations/connector-templates/source-low-code/secrets/config.json.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/main.py.hbs b/airbyte-integrations/connector-templates/source-low-code/src/main.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/main.py.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/main.py.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/__init__.py.hbs b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/__init__.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/__init__.py.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/__init__.py.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/manifest.yaml.hbs b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/manifest.yaml.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/manifest.yaml.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/manifest.yaml.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/run.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/run.py.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/run.py.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/TODO.md.hbs b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/TODO.md.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/TODO.md.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/TODO.md.hbs diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/customers.json b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/customers.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/customers.json rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/customers.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/employees.json b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/employees.json similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/schemas/employees.json rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/schemas/employees.json diff --git a/airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/source.py.hbs b/airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/source.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-configuration-based/src/source_{{snakeCase name}}/source.py.hbs rename to airbyte-integrations/connector-templates/source-low-code/src/source_{{snakeCase name}}/source.py.hbs diff --git a/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs b/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs deleted file mode 100644 index 28e5231c1cd1..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs +++ /dev/null @@ -1,105 +0,0 @@ -# {{capitalCase name}} Source - -This is the repository for the {{capitalCase name}} source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}). - -## Local development - -### Prerequisites - -* Python (`^3.9`) -* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) - - - -### Installing the connector - -From this connector directory, run: -```bash -poetry install --with dev -``` - - -### Create credentials - -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_{{snakeCase name}}/spec.yaml` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - - -### Locally running the connector - -``` -poetry run source-{{dashCase name}} spec -poetry run source-{{dashCase name}} check --config secrets/config.json -poetry run source-{{dashCase name}} discover --config secrets/config.json -poetry run source-{{dashCase name}} read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Running tests - -To run tests locally, from the connector directory run: - -``` -poetry run pytest tests -``` - -### Building the docker image - -1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) -2. Run the following command to build the docker image: -```bash -airbyte-ci connectors --name=source-{{dashCase name}} build -``` - -An image will be available on your host with the tag `airbyte/source-{{dashCase name}}:dev`. - - -### Running as a docker container - -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-{{dashCase name}}:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-{{dashCase name}}:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -### Running our CI test suite - -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): - -```bash -airbyte-ci connectors --name=source-{{dashCase name}} test -``` - -### Customizing acceptance Tests - -Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -### Dependency Management - -All of your dependencies should be managed via Poetry. -To add a new dependency, run: - -```bash -poetry add -``` - -Please commit the changes to `pyproject.toml` and `poetry.lock` files. - -## Publishing a new version of the connector - -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` -2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): - - bump the `dockerImageTag` value in in `metadata.yaml` - - bump the `version` value in `pyproject.toml` -3. Make sure the `metadata.yaml` content is up to date. -4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). -5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). -6. Pat yourself on the back for being an awesome contributor. -7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/abnormal_state.json b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/abnormal_state.json deleted file mode 100644 index 52b0f2c2118f..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/abnormal_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/acceptance.py deleted file mode 100644 index 9e6409236281..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/acceptance.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("connector_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/configured_catalog.json b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/configured_catalog.json deleted file mode 100644 index 36f0468db0d8..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/configured_catalog.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "customers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/invalid_config.json b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/invalid_config.json deleted file mode 100644 index f3732995784f..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_config.json b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_config.json deleted file mode 100644 index ecc4913b84c7..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_state.json b/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_state.json deleted file mode 100644 index 3587e579822d..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/sample_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs deleted file mode 100644 index 8d952455ab14..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs +++ /dev/null @@ -1,34 +0,0 @@ -data: - allowedHosts: - hosts: - - TODO # Please change to the hostname of the source. - registries: - oss: - enabled: true - cloud: - enabled: false - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-{{dashCase name}} - connectorBuildOptions: - # Please update to the latest version of the connector base image. - # https://hub.docker.com/r/airbyte/python-connector-base - # Please use the full address with sha256 hash to guarantee build reproducibility. - baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 - connectorSubtype: api - connectorType: source - definitionId: {{generateDefinitionId}} - dockerImageTag: 0.1.0 - dockerRepository: airbyte/source-{{dashCase name}} - githubIssueLabel: source-{{dashCase name}} - icon: {{dashCase name}}.svg - license: MIT - name: {{capitalCase name}} - releaseDate: TODO - supportLevel: community - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} - tags: - - language:python -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/pyproject.toml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/pyproject.toml.hbs deleted file mode 100644 index 40bbebd8c090..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/pyproject.toml.hbs +++ /dev/null @@ -1,27 +0,0 @@ -[build-system] -requires = [ "poetry-core>=1.0.0",] -build-backend = "poetry.core.masonry.api" - -[tool.poetry] -version = "0.1.0" -name = "source-{{dashCase name}}" -description = "Source implementation for {{dashCase name}}." -authors = [ "Airbyte ",] -license = "MIT" -readme = "README.md" -documentation = "https://docs.airbyte.com/integrations/sources/{{dashCase name}}" -homepage = "https://airbyte.com" -repository = "https://github.com/airbytehq/airbyte" -packages = [ { include = "source_{{snakeCase name}}", from="src"}, {include = "main.py", from = "src"} ] - -[tool.poetry.dependencies] -python = "^3.9,<3.12" -airbyte-cdk = "^0" - -[tool.poetry.scripts] -source-{{dashCase name}} = "source_{{snakeCase name}}.run:run" - -[tool.poetry.group.dev.dependencies] -requests-mock = "^1.9.3" -pytest-mock = "^3.6.1" -pytest = "^6.1" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/secrets/config.json.hbs b/airbyte-integrations/connector-templates/source-python-http-api/secrets/config.json.hbs deleted file mode 100644 index f5f8933895aa..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/secrets/config.json.hbs +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO populate with needed configuration for integration tests or delete this file and any references to it. The schema of this file should match what is in your spec.yaml" -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/main.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/src/main.py.hbs deleted file mode 100644 index 202f3973567d..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/src/main.py.hbs +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_{{snakeCase name}}.run import run - -if __name__ == "__main__": - run() diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/__init__.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/__init__.py.hbs deleted file mode 100644 index 09f02ce623ca..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/__init__.py.hbs +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from .source import Source{{properCase name}} - -__all__ = ["Source{{properCase name}}"] diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/run.py.hbs deleted file mode 100644 index 25c9400301f9..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/run.py.hbs +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from .source import Source{{properCase name}} - -def run(): - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/source.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/source.py.hbs deleted file mode 100644 index b6a9d584cb89..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/source.py.hbs +++ /dev/null @@ -1,206 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. -""" - - -# Basic full refresh stream -class {{properCase name}}Stream(HttpStream, ABC): - """ - TODO remove this comment - - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class {{properCase name}}Stream(HttpStream, ABC)` which is the current class - `class Customers({{properCase name}}Stream)` contains behavior to pull data for customers using v1/customers - `class Employees({{properCase name}}Stream)` contains behavior to pull data for employees using v1/employees - - If some streams implement incremental sync, it is typical to create another class - `class Incremental{{properCase name}}Stream(({{properCase name}}Stream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - - See the reference docs for the full list of configurable options. - """ - - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} - - -class Customers({{properCase name}}Stream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "customers" - - -# Basic incremental stream -class Incremental{{properCase name}}Stream({{properCase name}}Stream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} - - -class Employees(Incremental{{properCase name}}Stream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" - - def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. - """ - return "employees" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. - - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. - - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. - - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") - - -# Source -class Source{{properCase name}}(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - TODO: Replace the streams below with your own streams. - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/spec.yaml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/spec.yaml.hbs deleted file mode 100644 index 0ba7c6d552af..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/spec.yaml.hbs +++ /dev/null @@ -1,12 +0,0 @@ -documentationUrl: https://docsurl.com -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: {{capitalCase name}} Spec - type: object - required: - - TODO - properties: - # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': - TODO: - type: string - description: describe me diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py deleted file mode 100644 index c941b3045795..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connector-templates/source-python/README.md.hbs b/airbyte-integrations/connector-templates/source-python/README.md.hbs index 919b24fa5f35..28e5231c1cd1 100644 --- a/airbyte-integrations/connector-templates/source-python/README.md.hbs +++ b/airbyte-integrations/connector-templates/source-python/README.md.hbs @@ -67,7 +67,9 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ### Running our CI test suite + You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-{{dashCase name}} test ``` @@ -81,6 +83,7 @@ If your connector requires to create or destroy resources for use during accepta All of your dependencies should be managed via Poetry. To add a new dependency, run: + ```bash poetry add ``` diff --git a/airbyte-integrations/connector-templates/source-python/integration_tests/acceptance.py b/airbyte-integrations/connector-templates/source-python/integration_tests/acceptance.py index 43ce950d77ca..9e6409236281 100644 --- a/airbyte-integrations/connector-templates/source-python/integration_tests/acceptance.py +++ b/airbyte-integrations/connector-templates/source-python/integration_tests/acceptance.py @@ -11,6 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connector-templates/source-python/integration_tests/configured_catalog.json b/airbyte-integrations/connector-templates/source-python/integration_tests/configured_catalog.json index b999c2ba3abf..36f0468db0d8 100644 --- a/airbyte-integrations/connector-templates/source-python/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connector-templates/source-python/integration_tests/configured_catalog.json @@ -2,14 +2,21 @@ "streams": [ { "stream": { - "name": "table_name", + "name": "customers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": ["column_name"] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs index ba39befbfce3..8d952455ab14 100644 --- a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs @@ -2,15 +2,15 @@ data: allowedHosts: hosts: - TODO # Please change to the hostname of the source. - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-{{dashCase name}} registries: oss: enabled: true cloud: enabled: false + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-{{dashCase name}} connectorBuildOptions: # Please update to the latest version of the connector base image. # https://hub.docker.com/r/airbyte/python-connector-base diff --git a/airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs b/airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs index b494c8d9344b..f5f8933895aa 100644 --- a/airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs +++ b/airbyte-integrations/connector-templates/source-python/secrets/config.json.hbs @@ -1,3 +1,3 @@ { - "fix-me": "TODO populate with needed configuration for integration tests or delete this file and any references to it. The schema of this file should match what is in your spec.yaml" + "fix-me": "TODO populate with needed configuration for integration tests or delete this file and any references to it. The schema of this file should match what is in your spec.yaml" } diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/TODO.md b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/TODO.md similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/TODO.md rename to airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/TODO.md diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/customers.json b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/customers.json similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/customers.json rename to airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/customers.json diff --git a/airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/employees.json b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/employees.json similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/src/source_{{snakeCase name}}/schemas/employees.json rename to airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/schemas/employees.json diff --git a/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/source.py.hbs b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/source.py.hbs index 1e017d125218..b6a9d584cb89 100644 --- a/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/source.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/source.py.hbs @@ -3,103 +3,204 @@ # -import json -from datetime import datetime -from typing import Dict, Generator - -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteCatalog, - AirbyteConnectionStatus, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - Status, - Type, -) -from airbyte_cdk.sources import Source - - -class Source{{properCase name}}(Source): - def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the integration - e.g: if a provided Stripe API token can be used to connect to the Stripe API. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.yaml file - - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - try: - # Not Implemented - - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") - - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: - """ - Returns an AirbyteCatalog representing the available streams and fields in this integration. - For example, given valid credentials to a Postgres database, - returns an Airbyte catalog where each postgres table is a stream, and each table column is a field. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.yaml file - - :return: AirbyteCatalog is an object describing a list of all available streams in this source. - A stream is an AirbyteStream object that includes: - - its stream name (or table name in the case of Postgres) - - json_schema providing the specifications of expected schema for this stream (a list of columns described - by their names and types) - """ - streams = [] - - stream_name = "TableName" # Example - json_schema = { # Example - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": {"columnName": {"type": "string"}}, - } - - # Not Implemented - - streams.append(AirbyteStream(name=stream_name, json_schema=json_schema)) - return AirbyteCatalog(streams=streams) - - def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] - ) -> Generator[AirbyteMessage, None, None]: - """ - Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, - catalog, and state. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.yaml file - :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog - returned by discover(), but - in addition, it's been configured in the UI! For each particular stream and field, there may have been provided - with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc - :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume - replication in the future from that saved checkpoint. - This is the object that is provided with state from previous runs and avoid replicating the entire set of - data everytime. - - :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. - """ - stream_name = "TableName" # Example - data = {"columnName": "Hello World"} # Example - - # Not Implemented - - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream=stream_name, data=data, emitted_at=int(datetime.now().timestamp()) * 1000), - ) +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.yaml file. +""" + + +# Basic full refresh stream +class {{properCase name}}Stream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class {{properCase name}}Stream(HttpStream, ABC)` which is the current class + `class Customers({{properCase name}}Stream)` contains behavior to pull data for customers using v1/customers + `class Employees({{properCase name}}Stream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class Incremental{{properCase name}}Stream(({{properCase name}}Stream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Customers({{properCase name}}Stream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class Incremental{{properCase name}}Stream({{properCase name}}Stream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(Incremental{{properCase name}}Stream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class Source{{properCase name}}(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/spec.yaml.hbs b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/spec.yaml.hbs index 22a7ba6d7749..0ba7c6d552af 100644 --- a/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/spec.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python/src/source_{{snakeCase name}}/spec.yaml.hbs @@ -4,8 +4,9 @@ connectionSpecification: title: {{capitalCase name}} Spec type: object required: - - fix-me + - TODO properties: - fix-me: + # 'TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.': + TODO: type: string description: describe me diff --git a/airbyte-integrations/connector-templates/source-python-http-api/integration_tests/__init__.py b/airbyte-integrations/connector-templates/source-python/unit_tests/__init__.py similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/integration_tests/__init__.py rename to airbyte-integrations/connector-templates/source-python/unit_tests/__init__.py diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs b/airbyte-integrations/connector-templates/source-python/unit_tests/test_incremental_streams.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs rename to airbyte-integrations/connector-templates/source-python/unit_tests/test_incremental_streams.py.hbs diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs b/airbyte-integrations/connector-templates/source-python/unit_tests/test_source.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs rename to airbyte-integrations/connector-templates/source-python/unit_tests/test_source.py.hbs diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs b/airbyte-integrations/connector-templates/source-python/unit_tests/test_streams.py.hbs similarity index 100% rename from airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs rename to airbyte-integrations/connector-templates/source-python/unit_tests/test_streams.py.hbs diff --git a/airbyte-integrations/connector-templates/source-python/unit_tests/unit_test.py.hbs b/airbyte-integrations/connector-templates/source-python/unit_tests/unit_test.py.hbs deleted file mode 100644 index 219ae0142c72..000000000000 --- a/airbyte-integrations/connector-templates/source-python/unit_tests/unit_test.py.hbs +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index 9b156ceaf885..0bfecacc92d8 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -59,7 +59,6 @@ Search the generated directory for "TODO"s and follow them to implement your con - [Speedrun: Building a HTTP source with the CDK](tutorials/cdk-speedrun.md) - [Building a HTTP source with the CDK](tutorials/cdk-tutorial-python-http/getting-started.md) -- [Building a Python source](tutorials/building-a-python-source.md) - [Building a Java destination](tutorials/building-a-java-destination.md) As you implement your connector, make sure to review the [Best Practices for Connector Development](best-practices.md) guide. diff --git a/docs/connector-development/config-based/low-code-cdk-overview.md b/docs/connector-development/config-based/low-code-cdk-overview.md index da246f031775..c22efdc16cb2 100644 --- a/docs/connector-development/config-based/low-code-cdk-overview.md +++ b/docs/connector-development/config-based/low-code-cdk-overview.md @@ -67,7 +67,6 @@ If the answer to all questions is yes, you can use the low-code framework to bui - An API key for the source you want to build a connector for - Python >= 3.9 - Docker -- NodeJS ## Overview of the process diff --git a/docs/connector-development/tutorials/cdk-speedrun.md b/docs/connector-development/tutorials/cdk-speedrun.md index 35a9543d2e53..8cd1022d81cc 100644 --- a/docs/connector-development/tutorials/cdk-speedrun.md +++ b/docs/connector-development/tutorials/cdk-speedrun.md @@ -14,8 +14,7 @@ detail, check it out below. 1. Python >= 3.9 2. [Poetry](https://python-poetry.org/) -2. Docker -3. NodeJS +3. Docker #### Generate the Template @@ -27,7 +26,7 @@ cd airbyte-integrations/connector-templates/generator ./generate.sh ``` -Select the `Python HTTP API Source` and name it `python-http-example`. +Select the `Python CDK Source` and name it `python-http-example`. #### Create Dev Environment diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/creating-the-source.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/creating-the-source.md index ed4ff875bc38..8101cfe7e444 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/creating-the-source.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/creating-the-source.md @@ -9,7 +9,7 @@ $ ./generate.sh ``` This will bring up an interactive helper application. Use the arrow keys to pick a template from the -list. Select the `Python HTTP API Source` template and then input the name of your connector. The +list. Select the `Python CDK Source` template and then input the name of your connector. The application will create a new directory in airbyte/airbyte-integrations/connectors/ with the name of your new connector. diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md index f97c65bd6352..d8a5fdbeca2f 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md @@ -7,9 +7,10 @@ API. We'll be using the Exchange Rates API as an example since it is simple and of the capabilities of the CDK. ## Requirements -* Python >= 3.9 -* [Poetry](https://python-poetry.org/) -* Docker + +- Python 3.9 and higher +- [Poetry](https://python-poetry.org/) +- Docker All the commands below assume that `python` points to a version of python >=3.9.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the diff --git a/docs/contributing-to-airbyte/resources/developing-locally.md b/docs/contributing-to-airbyte/resources/developing-locally.md index ba99300b41c8..7bffa0174eb8 100644 --- a/docs/contributing-to-airbyte/resources/developing-locally.md +++ b/docs/contributing-to-airbyte/resources/developing-locally.md @@ -3,7 +3,7 @@ The following technologies are required to build Airbyte locally. 1. [`Java 21`](https://jdk.java.net/archive/) -2. `Node 16` +2. `Node 20.` 3. `Python 3.9` 4. `Docker` 5. `Jq`