From b85c51304f23889f11b9f7482cb02ee2b738cdbf Mon Sep 17 00:00:00 2001 From: "Sherif A. Nada" Date: Mon, 11 Apr 2022 20:12:06 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Update=20hardway=20tutorial=20&?= =?UTF-8?q?=20generic=20source=20template=20to=20use=20SAT=20(#11908)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source_acceptance_test/tests/test_core.py | 4 +- .../utils/json_schema_helper.py | 2 +- .../connector-templates/generator/plopfile.js | 6 - .../source-generic/.gitignore.hbs | 1 - .../source-generic/Dockerfile | 2 +- .../NEW_SOURCE_CHECKLIST.md.hbs | 15 -- .../{README.md.hbs => README.md} | 0 .../source-generic/acceptance-test-config.yml | 25 +++ .../source-generic/acceptance-test-docker.sh | 16 ++ .../source-generic/build.gradle | 15 +- .../airbyte-source-acceptance-test.gradle | 5 + .../build-a-connector-the-hard-way.md | 142 +++++++++--------- 12 files changed, 124 insertions(+), 109 deletions(-) delete mode 100644 airbyte-integrations/connector-templates/source-generic/.gitignore.hbs delete mode 100644 airbyte-integrations/connector-templates/source-generic/NEW_SOURCE_CHECKLIST.md.hbs rename airbyte-integrations/connector-templates/source-generic/{README.md.hbs => README.md} (100%) create mode 100644 airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml create mode 100644 airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 251eb237f653..0d07d8773b34 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -256,8 +256,8 @@ def _validate_records_structure(records: List[AirbyteRecordMessage], configured_ just running schema validation is not enough case schema could have additionalProperties parameter set to true and no required fields therefore any arbitrary object would pass schema validation. - This method is here to catch those cases by extracting all the pathes - from the object and compare it to pathes expected from jsonschema. If + This method is here to catch those cases by extracting all the paths + from the object and compare it to paths expected from jsonschema. If there no common pathes then raise an alert. :param records: List of airbyte record messages gathered from connector instances. diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py index 25ee1b7779e0..ad350173e9bb 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py @@ -168,7 +168,7 @@ def _traverse_obj_and_get_path(obj, path=""): def get_expected_schema_structure(schema: dict, annotate_one_of: bool = False) -> List[str]: """ - Travers through json schema and compose list of property keys that object expected to have. + Traverse through json schema and compose list of property keys that object expected to have. :param annotate_one_of: Generate one_of index in path :param schema: jsonschema to get expected paths :returns list of object property keys paths diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index d2b0ae02a14c..86c5f2157552 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -175,12 +175,6 @@ module.exports = function (plop) { base: genericSourceInputRoot, templateFiles: `${genericSourceInputRoot}/**/**`, }, - { - type:'add', - abortOnFail: true, - templateFile: `${genericSourceInputRoot}/.gitignore.hbs`, - path: `${genericSourceOutputRoot}/.gitignore` - }, {type: 'emitSuccess', outputPath: genericSourceOutputRoot} ] }); diff --git a/airbyte-integrations/connector-templates/source-generic/.gitignore.hbs b/airbyte-integrations/connector-templates/source-generic/.gitignore.hbs deleted file mode 100644 index 29fffc6a50cc..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/.gitignore.hbs +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connector-templates/source-generic/Dockerfile b/airbyte-integrations/connector-templates/source-generic/Dockerfile index cccd14b23c42..846e34ccc961 100644 --- a/airbyte-integrations/connector-templates/source-generic/Dockerfile +++ b/airbyte-integrations/connector-templates/source-generic/Dockerfile @@ -1,7 +1,7 @@ FROM scratch ## TODO Add your dockerfile instructions here -## TODO uncomment the below line +## TODO uncomment the below line. This is required for Kubernetes compatibility. # ENV AIRBYTE_ENTRYPOINT="update this with the command you use for an entrypoint" # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. diff --git a/airbyte-integrations/connector-templates/source-generic/NEW_SOURCE_CHECKLIST.md.hbs b/airbyte-integrations/connector-templates/source-generic/NEW_SOURCE_CHECKLIST.md.hbs deleted file mode 100644 index 573f5454cd11..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/NEW_SOURCE_CHECKLIST.md.hbs +++ /dev/null @@ -1,15 +0,0 @@ -# New generic source checklist - -This is an autogenerated file describing the steps needed to implement a new Airbyte source. - -- Make sure you've read the [Airbyte Specification documentation](https://docs.airbyte.io/architecture/airbyte-specification) -- Iterate on your connector locally in your language of choice until it works as intended -- Edit the Dockerfile so the image it produces contains your connector and conforms to the Airbyte Protocol -- Once your image is working, make sure your connector passes the standard Airbyte test suite by running - `./gradlew :airbyte-integrations:connectors:source-{{snakeCase name}}:standardSourceTestFile`. See - [Testing Connectors](https://docs.airbyte.io/connector-development/testing-connectors) for more information about - standard tests. -- If API credentials are required to run the integration, please document how they can be obtained or link to a how-to guide. - -At any point during this process, feel free to create a PR against the Airbyte Repo and ask for any support or questions. We're always here to help! - diff --git a/airbyte-integrations/connector-templates/source-generic/README.md.hbs b/airbyte-integrations/connector-templates/source-generic/README.md similarity index 100% rename from airbyte-integrations/connector-templates/source-generic/README.md.hbs rename to airbyte-integrations/connector-templates/source-generic/README.md diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml b/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml new file mode 100644 index 000000000000..4d2eeb3e00e6 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-generic/acceptance-test-config.yml @@ -0,0 +1,25 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-{{dashCase name}}:dev +tests: + spec: + - spec_path: "spec.json" + config_path: "secrets/valid_config.json" # TODO add this file + connection: + - config_path: "secrets/valid_config.json" # TODO add this file + status: "succeed" + - config_path: "secrets/invalid_config.json" # TODO add this file + status: "failed" + discovery: + - config_path: "secrets/valid_config.json" + basic_read: + - config_path: "secrets/valid_config.json" + configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file + empty_streams: [] + full_refresh: + - config_path: "secrets/valid_config.json" + configured_catalog_path: "fullrefresh_configured_catalog.json" # TODO add or change this file +# incremental: # TODO uncomment this once you implement incremental sync +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-generic/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connector-templates/source-generic/build.gradle b/airbyte-integrations/connector-templates/source-generic/build.gradle index 4e2ca579499b..af82fb4a3cc0 100644 --- a/airbyte-integrations/connector-templates/source-generic/build.gradle +++ b/airbyte-integrations/connector-templates/source-generic/build.gradle @@ -1,17 +1,6 @@ plugins { // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image - // via ./gradlew :airbyte-integrations:connectors:source-{{snakeCase name}}:build + // via ./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbyteStandardSourceTestFile { - // All these input paths must live inside this connector's directory (or subdirectories) - configPath = "" // TODO - configuredCatalogPath = "" // TODO - specPath = "" // TODO -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) + id 'airbyte-source-acceptance-test' } diff --git a/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle b/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle index 4195d570557b..658460e5a8d1 100644 --- a/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle +++ b/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle @@ -35,6 +35,11 @@ class AirbyteSourceAcceptanceTestPlugin implements Plugin { project.sourceAcceptanceTest.dependsOn(project.airbyteDockerTest) } + // make sure we create the integrationTest task once + if (!project.tasks.findByName('integrationTest')) { + project.task('integrationTest') + } + project.integrationTest.dependsOn(project.sourceAcceptanceTest) } } diff --git a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md index c15f05d0b0dd..a781e0faa435 100644 --- a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md +++ b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md @@ -24,7 +24,7 @@ To run this tutorial, you'll need: * Docker, Python, and Java with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md). * The `requests` Python package installed via `pip install requests` \(or `pip3` if `pip` is linked to a Python2 installation on your system\) -**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.7. Verify this by running +**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.7 or greater. Verify this by running ```bash $ python --version @@ -88,7 +88,7 @@ Head to the connector directory and we should see the following files have been ```bash $ cd ../../connectors/source-stock-ticker-api $ ls -Dockerfile NEW_SOURCE_CHECKLIST.md build.gradle +Dockerfile README.md acceptance-test-config.yml acceptance-test-docker.sh build.gradle ``` We'll use each of these files later. But first, let's write some code! @@ -153,9 +153,10 @@ We'll save this file in the root directory of our connector. Now we have the fol ```bash $ ls -1 Dockerfile -NEW_SOURCE_CHECKLIST.md +README.md +acceptance-test-config.yml +acceptance-test-docker.sh build.gradle -source.py spec.json ``` @@ -494,31 +495,33 @@ First, let's create a configured catalog `fullrefresh_configured_catalog.json` t ```javascript { - "streams": [ - { - "stream": { - "name": "stock_prices", - "supported_sync_modes": [ - "full_refresh" - ], - "json_schema": { - "properties": { - "date": { - "type": "string" - }, - "price": { - "type": "number" + "streams": [ + { + "stream": { + "name": "stock_prices", + "supported_sync_modes": [ + "full_refresh" + ], + "json_schema": { + "properties": { + "date": { + "type": "string" + }, + "price": { + "type": "number" + }, + "stock_ticker": { + "type": "string" + } + } + } }, - "stock_ticker": { - "type": "string" - } - } + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } - }, - "sync_mode": "full_refresh" - } - ] + ] } + ``` Then we'll define the `read` method in `source.py`: @@ -933,56 +936,61 @@ $ docker run -v $(pwd)/secrets/valid_config.json:/data/config.json -v $(pwd)/ful {'type': 'RECORD', 'record': {'stream': 'stock_prices', 'data': {'date': '2020-12-21', 'stock_ticker': 'TSLA', 'price': 649.86}, 'emitted_at': 1608628424000}} ``` -and with that, we've packaged our connector in a functioning Docker image. The last requirement before calling this connector finished is to pass the [Airbyte Standard Test suite](../testing-connectors/source-acceptance-tests-reference.md). +and with that, we've packaged our connector in a functioning Docker image. The last requirement before calling this connector finished is to pass the [Airbyte Source Acceptance Test suite](../testing-connectors/source-acceptance-tests-reference.md). ### 4. Test the connector -The minimum requirement for testing our connector is to pass the Airbyte Standard Test suite. You're encouraged to add custom test cases for your connector where it makes sense to do so e.g: to test edge cases that are not covered by the standard suite. But at the very least, you must pass Airbyte's Standard Test suite. - -To integrate with the standard test suite, modify the generated `build.gradle` file as follows: - -```groovy -plugins { - // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image - // via ./gradlew :airbyte-integrations:connectors:source-stock-ticker-api:build - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbyteStandardSourceTestFile { - // All these input paths must live inside this connector's directory (or subdirectories) - configPath = "secrets/valid_config.json" - configuredCatalogPath = "fullrefresh_configured_catalog.json" - specPath = "spec.json" -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) -} +The minimum requirement for testing your connector is to pass the [Source Acceptance Test (SAT)](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) suite. SAT is a blackbox test suite containing a number of tests that validate your connector behaves as intended by the Airbyte Specification. You're encouraged to add custom test cases for your connector where it makes sense to do so e.g: to test edge cases that are not covered by the standard suite. But at the very least, you must pass Airbyte's SATs suite. + +The code generator should have already generated a YAML file which configures the test suite. In order to run it, modify the `acceptance-test-config.yaml` file to look like this: + + +```yaml +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-stock-ticker-api:dev +tests: + spec: + - spec_path: "spec.json" + config_path: "secrets/valid_config.json" + connection: + - config_path: "secrets/valid_config.json" + status: "succeed" + - config_path: "secrets/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/valid_config.json" + basic_read: + - config_path: "secrets/valid_config.json" + configured_catalog_path: "fullrefresh_configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/valid_config.json" + configured_catalog_path: "fullrefresh_configured_catalog.json" +# incremental: # TODO uncomment this once you implement incremental sync in part 2 of the tutorial +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" ``` -Then **from the Airbyte repository root**, run: +Then from the connector module directory run ```bash -./gradlew clean :airbyte-integrations:connectors:source-stock-ticker-api:integrationTest +./acceptance-test-docker.sh ``` After tests have run, you should see a test summary like: ```text -Test run finished after 5049 ms -[ 2 containers found ] -[ 0 containers skipped ] -[ 2 containers started ] -[ 0 containers aborted ] -[ 2 containers successful ] -[ 0 containers failed ] -[ 7 tests found ] -[ 0 tests skipped ] -[ 7 tests started ] -[ 0 tests aborted ] -[ 7 tests successful ] -[ 0 tests failed ] +collecting ... + test_core.py ✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓ 95% █████████▌ + test_full_refresh.py ✓ 100% ██████████ + +================== short test summary info ================== +SKIPPED [1] source_acceptance_test/plugin.py:56: Skipping TestIncremental.test_two_sequential_reads because not found in the config + +Results (8.91s): + 20 passed ``` That's it! We've created a fully functioning connector. Now let's get to the exciting part: using it from the Airbyte UI. @@ -1034,12 +1042,6 @@ If the Airbyte server isn't already running, start it by running **from the Airb docker-compose up ``` ->**_NOTE:_** if you are on an M1 Mac, you have to build Airbyte first and then run it in `dev` mode: ->```bash ->SUB_BUILD=PLATFORM ./gradlew build ->VERSION=dev docker-compose up ->``` - When Airbyte server is done starting up, it prints the following banner in the log output \(it can take 10-20 seconds for the server to start\): ```bash