From afc399e9c2593ec0a0b6dc8e75fd6239f18690e3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 9 Jan 2024 13:12:36 +0100 Subject: [PATCH 01/13] airbyte-lib testing --- .../pipelines/airbyte_ci/connectors/consts.py | 1 + .../test/steps/python_connectors.py | 106 ++++++++++++++---- airbyte-ci/connectors/pipelines/poetry.lock | 35 +++--- .../connectors/pipelines/pyproject.toml | 1 + 4 files changed, 99 insertions(+), 44 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py index 10e00bc67dad..349b52cbe597 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py @@ -14,6 +14,7 @@ class CONNECTOR_TEST_STEP_ID(str, Enum): BUILD = "build" CHECK_BASE_IMAGE = "check_base_image" INTEGRATION = "integration" + AIRBYTE_LIB_VALIDATION = "airbyte_lib_validation" METADATA_VALIDATION = "metadata_validation" QA_CHECKS = "qa_checks" UNIT = "unit" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 65117b6d804b..4a6110833d98 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -5,8 +5,9 @@ """This module groups steps made to run tests for a specific Python connector given a test context.""" from abc import ABC, abstractmethod -from typing import List, Sequence, Tuple +from typing import Callable, List, Sequence, Tuple +import dpath.util import pipelines.dagger.actions.python.common import pipelines.dagger.actions.system.docker from dagger import Container, File @@ -188,6 +189,61 @@ def additional_pytest_options(self) -> List[str]: return super().additional_pytest_options + coverage_options +class AirbyteLibValidation(Step): + """A step to validate the connector will work with airbyte-lib, using the airbyte-lib validation helper.""" + + title = "AirbyteLib validation tests" + + async def _run(self, connector_under_test: Container) -> StepResult: + """Run all pytest tests declared in the test directory of the connector code. + Args: + connector_under_test (Container): The connector under test container. + Returns: + StepResult: Failure or success of the unit tests with stdout and stdout. + """ + context: ConnectorContext = self.context + # TODO skip if not pypi published + if False and dpath.util.get(context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: + return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") + + test_environment = await self.install_testing_environment(connector_under_test) + + connector_secrets = await self.context.get_connector_secrets() + first_secret_name = list(connector_secrets.keys())[0] + test_execution = test_environment.with_exec( + ["airbyte-lib-validate-source", "--connector-dir", ".", "--sample-config", "secrets/" + first_secret_name] + ) + + return await self.get_step_result(test_execution) + + async def install_testing_environment( + self, + built_connector_container: Container, + ) -> Callable: + """Add airbyte-lib and secrets to the test environment.""" + context: ConnectorContext = self.context + secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets") + + container_with_test_deps = await pipelines.dagger.actions.python.common.with_python_package( + self.context, built_connector_container.with_entrypoint([]), str(context.connector.code_directory) + ) + return ( + container_with_test_deps.with_exec( + [ + "pip", + "install", + "--index-url", + "https://test.pypi.org/simple/", + "--extra-index-url", + "https://pypi.org/simple", + "airbyte-lib", + ] + ) + # Mount the secrets + .with_(secret_mounting_function).with_env_variable("PYTHONPATH", ".") + ) + + class IntegrationTests(PytestStep): """A step to run the connector integration tests with Pytest.""" @@ -202,31 +258,37 @@ def get_test_steps(context: ConnectorContext) -> STEP_TREE: """ return [ [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD, step=BuildConnectorImages(context))], + # [ + # StepToRun( + # id=CONNECTOR_TEST_STEP_ID.UNIT, + # step=UnitTests(context), + # args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + # ) + # ], [ + # StepToRun( + # id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + # step=IntegrationTests(context), + # args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + # ), StepToRun( - id=CONNECTOR_TEST_STEP_ID.UNIT, - step=UnitTests(context), + id=CONNECTOR_TEST_STEP_ID.AIRBYTE_LIB_VALIDATION, + step=AirbyteLibValidation(context), args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - ) - ], - [ - StepToRun( - id=CONNECTOR_TEST_STEP_ID.INTEGRATION, - step=IntegrationTests(context), - args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, - depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - ), - StepToRun( - id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, - step=AcceptanceTests(context, context.concurrent_cat), - args=lambda results: { - "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] - }, - depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - ), - StepToRun( - id=CONNECTOR_TEST_STEP_ID.CHECK_BASE_IMAGE, step=CheckBaseImageIsUsed(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD] ), + # StepToRun( + # id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + # step=AcceptanceTests(context, context.concurrent_cat), + # args=lambda results: { + # "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] + # }, + # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + # ), + # StepToRun( + # id=CONNECTOR_TEST_STEP_ID.CHECK_BASE_IMAGE, step=CheckBaseImageIsUsed(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD] + # ), ], ] diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index a1e1a4468abe..037b485ce055 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-connectors-base-images" @@ -646,6 +646,17 @@ websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.3)"] +[[package]] +name = "dpath" +version = "2.1.6" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.1.6-py3-none-any.whl", hash = "sha256:31407395b177ab63ef72e2f6ae268c15e938f2990a8ecf6510f5686c02b6db73"}, + {file = "dpath-2.1.6.tar.gz", hash = "sha256:f1e07c72e8605c6a9e80b64bc8f42714de08a789c7de417e49c3f87a19692e47"}, +] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -1134,16 +1145,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1983,7 +1984,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1991,15 +1991,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2016,7 +2009,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2024,7 +2016,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2531,4 +2522,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "c24e2d2a6c1288b7fe7efa1c0ca598dfef8942bb437986512bbe17bcf1408799" +content-hash = "bf876f28b303f1db188a7dcacfc91911f9c7b5b81e849a21ab1d289501cf346f" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index faf30e9fc8ad..492c9d65842f 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -27,6 +27,7 @@ segment-analytics-python = "^2.2.3" pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" +dpath = "^2.1.6" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 1b16bb6e45ff3a0f2cecc8e8129486d1aaf6ffed Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 18:00:21 +0100 Subject: [PATCH 02/13] format --- .../test/steps/python_connectors.py | 7 +- .../test_tests/test_python_connectors.py | 73 ++++++++++++++++++- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index fd025a088de7..c9cb50084c7e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -11,6 +11,7 @@ import pipelines.dagger.actions.python.common import pipelines.dagger.actions.system.docker from dagger import Container, File +from pipelines import hacks from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID from pipelines.airbyte_ci.connectors.context import ConnectorContext @@ -211,8 +212,10 @@ async def _run(self, connector_under_test: Container) -> StepResult: connector_secrets = await self.context.get_connector_secrets() first_secret_name = list(connector_secrets.keys())[0] - test_execution = test_environment.with_exec( - ["airbyte-lib-validate-source", "--connector-dir", ".", "--sample-config", "secrets/" + first_secret_name] + test_execution = test_environment.with_( + hacks.never_fail_exec( + ["airbyte-lib-validate-source", "--connector-dir", ".", "--sample-config", "secrets/" + first_secret_name] + ) ) return await self.get_step_result(test_execution) diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index 2d89af9ec94d..53e194488734 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -6,8 +6,8 @@ from connector_ops.utils import Connector, ConnectorLanguage from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext -from pipelines.airbyte_ci.connectors.test.steps.python_connectors import UnitTests -from pipelines.models.steps import StepResult +from pipelines.airbyte_ci.connectors.test.steps.python_connectors import AirbyteLibValidation, UnitTests +from pipelines.models.steps import StepResult, StepStatus pytestmark = [ pytest.mark.anyio, @@ -105,3 +105,72 @@ def test_params(self, context_for_certified_connector_with_setup): f"--cov={context_for_certified_connector_with_setup.connector.technical_name.replace('-', '_')}", f"--cov-fail-under={step.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS}", ] + + +class TestAirbyteLibValidationTests: + @pytest.fixture + def compatible_connector(self): + return Connector("source-faker") + + @pytest.fixture + def valid_secrets(self, dagger_client): + return { + "valid_config.json": dagger_client.set_secret( + "faker_secret_config", + """{ + "count": 10, + "seed": 0, + "parallelism": 1, + "always_updated": false +}""", + ) + } + + @pytest.fixture + def invalid_secrets(self, dagger_client): + return { + "invalid_config.json": dagger_client.set_secret( + "faker_secret_config", + """{ + "count": "a string even though it should be a number" +}""", + ) + } + + @pytest.fixture + def context_for_valid_connector(self, mocker, compatible_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test airbyte-lib validation", + connector=compatible_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + context.get_connector_secrets = mocker.AsyncMock(return_value={}) + return context + + @pytest.fixture + async def connector_container(self, context_for_valid_connector, current_platform): + result = await BuildConnectorImages(context_for_valid_connector).run() + return result.output_artifact[current_platform] + + async def test__run_validation_success(self, mocker, valid_secrets, context_for_valid_connector: ConnectorContext, connector_container): + context_for_valid_connector.get_connector_secrets = mocker.AsyncMock(return_value=valid_secrets) + result = await AirbyteLibValidation(context_for_valid_connector)._run(connector_container) + assert isinstance(result, StepResult) + assert result.status == StepStatus.SUCCESS + assert "Creating source and validating spec and version..." in result.stdout + assert "Running check..." in result.stdout + assert "Fetching streams..." in result.stdout + assert "Trying to peek stream" in result.stdout + + async def test__run_validation_fail(self, mocker, invalid_secrets, context_for_valid_connector: ConnectorContext, connector_container): + context_for_valid_connector.get_connector_secrets = mocker.AsyncMock(return_value=invalid_secrets) + result = await AirbyteLibValidation(context_for_valid_connector)._run(connector_container) + assert isinstance(result, StepResult) + assert result.status == StepStatus.FAILURE + assert "'a string even though it should be a number' is not of type 'integer'" in result.stderr From 8d233beca628d791b172f1929a00b896125cd289 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 25 Jan 2024 17:47:59 +0100 Subject: [PATCH 03/13] revert changes --- airbyte-lib/poetry.lock | 6 +++--- poetry.lock | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock index cf96e123e0a2..5dafb19e798a 100644 --- a/airbyte-lib/poetry.lock +++ b/airbyte-lib/poetry.lock @@ -1690,13 +1690,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no [[package]] name = "pytest-docker" -version = "2.1.0" +version = "2.0.1" description = "Simple pytest fixtures for Docker and Docker Compose based tests" optional = false python-versions = ">=3.6" files = [ - {file = "pytest-docker-2.1.0.tar.gz", hash = "sha256:31d834d5788baa88e3adbcbdc802a01133af83fcaeb58220af6c3f0adb341025"}, - {file = "pytest_docker-2.1.0-py3-none-any.whl", hash = "sha256:6357207a760452257476911458f51b9e9df194e76d7f59cf94728b3437534e5b"}, + {file = "pytest-docker-2.0.1.tar.gz", hash = "sha256:1c17e9202a566f85ed5ef269fe2815bd4899e90eb639622e5d14277372ca7524"}, + {file = "pytest_docker-2.0.1-py3-none-any.whl", hash = "sha256:7103f97b8c479c826b63d73cfb83383dc1970d35105ed1ce78a722c90c7fe650"}, ] [package.dependencies] diff --git a/poetry.lock b/poetry.lock index 50473ab6235e..7e3ee153c35a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "black" @@ -99,24 +99,24 @@ files = [ [[package]] name = "pathspec" -version = "0.12.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] [[package]] name = "platformdirs" -version = "4.1.0" +version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, - {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, ] [package.extras] From fef012c348055b76ec523bfab7f73e14d6a8fc32 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 17:42:58 +0100 Subject: [PATCH 04/13] fix up and test --- .../test/steps/python_connectors.py | 57 +++++++++---------- .../connectors/source-pokeapi/Dockerfile | 2 +- .../connectors/source-pokeapi/main.py | 1 + .../connectors/source-pokeapi/metadata.yaml | 2 +- docs/integrations/sources/pokeapi.md | 3 +- 5 files changed, 31 insertions(+), 34 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index c9cb50084c7e..bfbc42ed9187 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -204,8 +204,7 @@ async def _run(self, connector_under_test: Container) -> StepResult: StepResult: Failure or success of the unit tests with stdout and stdout. """ context: ConnectorContext = self.context - # TODO skip if not pypi published - if False and dpath.util.get(context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: + if dpath.util.get(context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") test_environment = await self.install_testing_environment(connector_under_test) @@ -236,10 +235,6 @@ async def install_testing_environment( [ "pip", "install", - "--index-url", - "https://test.pypi.org/simple/", - "--extra-index-url", - "https://pypi.org/simple", "airbyte-lib", ] ) @@ -262,37 +257,37 @@ def get_test_steps(context: ConnectorContext) -> STEP_TREE: """ return [ [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD, step=BuildConnectorImages(context))], - # [ - # StepToRun( - # id=CONNECTOR_TEST_STEP_ID.UNIT, - # step=UnitTests(context), - # args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, - # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - # ) - # ], [ - # StepToRun( - # id=CONNECTOR_TEST_STEP_ID.INTEGRATION, - # step=IntegrationTests(context), - # args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, - # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - # ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.UNIT, + step=UnitTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ) + ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), StepToRun( id=CONNECTOR_TEST_STEP_ID.AIRBYTE_LIB_VALIDATION, step=AirbyteLibValidation(context), args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], ), - # StepToRun( - # id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, - # step=AcceptanceTests(context, context.concurrent_cat), - # args=lambda results: { - # "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] - # }, - # depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], - # ), - # StepToRun( - # id=CONNECTOR_TEST_STEP_ID.CHECK_BASE_IMAGE, step=CheckBaseImageIsUsed(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD] - # ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, context.concurrent_cat), + args=lambda results: { + "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] + }, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.CHECK_BASE_IMAGE, step=CheckBaseImageIsUsed(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD] + ), ], ] diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 0d27d3737d6f..9bf098140d9a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -34,5 +34,5 @@ COPY source_pokeapi ./source_pokeapi ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index f32ce6b38105..469ba443e8ac 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -5,4 +5,5 @@ from source_pokeapi.run import run if __name__ == "__main__": + # dummy change run() diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 2f6a410ea49b..13d757cdc0bd 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -14,7 +14,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index ee543b33e024..44939f5b900d 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -36,7 +36,8 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | -| 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code +| 0.2.1 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Test change | +| 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code | | 0.1.5 | 2022-05-18 | [12942](https://github.com/airbytehq/airbyte/pull/12942) | Fix example inputs | | 0.1.4 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | | 0.1.3 | 2021-12-03 | [8432](https://github.com/airbytehq/airbyte/pull/8432) | Migrate from base_python to CDK, add SAT tests. | From 2b159a36f56bdfff9231badd499af906a910029b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 17:47:41 +0100 Subject: [PATCH 05/13] fix poetry lockfile --- airbyte-ci/connectors/pipelines/poetry.lock | 10 ++-------- airbyte-ci/connectors/pipelines/pyproject.toml | 6 +----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 61712cc1b114..606704d4f78f 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -648,7 +648,6 @@ websocket-client = ">=0.32.0" ssh = ["paramiko (>=2.4.3)"] [[package]] -<<<<<<< HEAD name = "dpath" version = "2.1.6" description = "Filesystem-like pathing and searching for dictionaries" @@ -659,7 +658,7 @@ files = [ {file = "dpath-2.1.6.tar.gz", hash = "sha256:f1e07c72e8605c6a9e80b64bc8f42714de08a789c7de417e49c3f87a19692e47"}, ] -======= +[[package]] name = "editor" version = "1.6.5" description = "🖋 Open the default text editor 🖋" @@ -674,7 +673,6 @@ files = [ runs = "*" xmod = "*" ->>>>>>> origin/master [[package]] name = "exceptiongroup" version = "1.2.0" @@ -2589,8 +2587,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -<<<<<<< HEAD -content-hash = "bf876f28b303f1db188a7dcacfc91911f9c7b5b81e849a21ab1d289501cf346f" -======= -content-hash = "de011a00b912c9acd6d4f202c7cf54308010efc14a214ee25608712851306aa9" ->>>>>>> origin/master +content-hash = "68c67e470fdf10ce89ada657bbbaa94dfbad98324b19d866de9cbb924beab5f7" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index ac1c96acb8dc..d2f3a8824748 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -27,14 +27,10 @@ segment-analytics-python = "^2.2.3" pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" -dpath = "^2.1.6" tomli = "^2.0.1" tomli-w = "^1.0.0" types-requests = "2.28.2" - -[tool.poetry.group.test.dependencies] -pytest = "^6.2.5" -pytest-mock = "^3.10.0" +dpath = "^2.1.6" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" From e7045c9b266d75f6235919258cf46c5909bb5ffb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 18:50:25 +0100 Subject: [PATCH 06/13] fix --- .../connectors/test/steps/python_connectors.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index bfbc42ed9187..acd601c1b1fa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -18,6 +18,7 @@ from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.dagger.actions import secrets +from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun from pipelines.models.steps import STEP_PARAMS, Step, StepResult @@ -196,6 +197,8 @@ class AirbyteLibValidation(Step): title = "AirbyteLib validation tests" + context: ConnectorContext + async def _run(self, connector_under_test: Container) -> StepResult: """Run all pytest tests declared in the test directory of the connector code. Args: @@ -203,11 +206,10 @@ async def _run(self, connector_under_test: Container) -> StepResult: Returns: StepResult: Failure or success of the unit tests with stdout and stdout. """ - context: ConnectorContext = self.context - if dpath.util.get(context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: + if dpath.util.get(self.context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") - test_environment = await self.install_testing_environment(connector_under_test) + test_environment = await self.install_testing_environment(with_poetry(self.context)) connector_secrets = await self.context.get_connector_secrets() first_secret_name = list(connector_secrets.keys())[0] @@ -222,7 +224,7 @@ async def _run(self, connector_under_test: Container) -> StepResult: async def install_testing_environment( self, built_connector_container: Container, - ) -> Callable: + ) -> Container: """Add airbyte-lib and secrets to the test environment.""" context: ConnectorContext = self.context secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets") From 833a4ea5ccfdbe1289270d315b9db7ed4d8c8e37 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 18:57:18 +0100 Subject: [PATCH 07/13] format --- .../airbyte_ci/connectors/test/steps/python_connectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index acd601c1b1fa..0ae61a61a877 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -5,7 +5,7 @@ """This module groups steps made to run tests for a specific Python connector given a test context.""" from abc import ABC, abstractmethod -from typing import Callable, List, Sequence, Tuple +from typing import List, Sequence, Tuple import dpath.util import pipelines.dagger.actions.python.common From 3f58cce7257c66f607b4fdfd1a81a0f920db5a97 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 19:07:12 +0100 Subject: [PATCH 08/13] cleanup --- airbyte-ci/connectors/pipelines/README.md | 3 +++ airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- airbyte-integrations/connectors/source-pokeapi/Dockerfile | 2 +- airbyte-integrations/connectors/source-pokeapi/main.py | 1 - airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 2 +- docs/integrations/sources/pokeapi.md | 1 - 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 20490091b520..6c421043560e 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -261,6 +261,7 @@ flowchart TD build[Build connector docker image] unit[Run unit tests] integration[Run integration tests] + airbyte_lib_validation[Run airbyte-lib validation tests] cat[Run connector acceptance tests] secret[Load connector configuration] @@ -268,6 +269,7 @@ flowchart TD unit-->build secret-->integration secret-->cat + secret-->airbyte_lib_validation build-->integration build-->cat end @@ -610,6 +612,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 3.10.1 | [#34044](https://github.com/airbytehq/airbyte/pull/34044) | Add pypi validation testing. | | 3.10.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of separate check URL to check whether package exists already. | | 3.9.0 | [#34606](https://github.com/airbytehq/airbyte/pull/34606) | Allow configuration of python registry URL via environment variable. | | 3.8.1 | [#34607](https://github.com/airbytehq/airbyte/pull/34607) | Improve gradle dependency cache volume protection. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index d2f3a8824748..1f00c83eb4b3 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.10.0" +version = "3.10.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 9bf098140d9a..0d27d3737d6f 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -34,5 +34,5 @@ COPY source_pokeapi ./source_pokeapi ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 469ba443e8ac..f32ce6b38105 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -5,5 +5,4 @@ from source_pokeapi.run import run if __name__ == "__main__": - # dummy change run() diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 13d757cdc0bd..2f6a410ea49b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -14,7 +14,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index 44939f5b900d..42ed61bdf6af 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -36,7 +36,6 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | -| 0.2.1 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Test change | | 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code | | 0.1.5 | 2022-05-18 | [12942](https://github.com/airbytehq/airbyte/pull/12942) | Fix example inputs | | 0.1.4 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | From eed3a03f422ba1365c1d7c5c8391e2d590fa0d0e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 19:41:23 +0100 Subject: [PATCH 09/13] fix test --- .../pipelines/tests/test_tests/test_python_connectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index 53e194488734..d271543362f3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -166,7 +166,7 @@ async def test__run_validation_success(self, mocker, valid_secrets, context_for_ assert "Creating source and validating spec and version..." in result.stdout assert "Running check..." in result.stdout assert "Fetching streams..." in result.stdout - assert "Trying to peek stream" in result.stdout + assert "Trying to read from stream" in result.stdout async def test__run_validation_fail(self, mocker, invalid_secrets, context_for_valid_connector: ConnectorContext, connector_container): context_for_valid_connector.get_connector_secrets = mocker.AsyncMock(return_value=invalid_secrets) From 34b3a4cdac17f89312c1ce5d5040d74389024637 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 31 Jan 2024 19:43:09 +0100 Subject: [PATCH 10/13] revert --- docs/integrations/sources/pokeapi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index 42ed61bdf6af..ee543b33e024 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -36,7 +36,7 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | -| 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code | +| 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code | 0.1.5 | 2022-05-18 | [12942](https://github.com/airbytehq/airbyte/pull/12942) | Fix example inputs | | 0.1.4 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | | 0.1.3 | 2021-12-03 | [8432](https://github.com/airbytehq/airbyte/pull/8432) | Migrate from base_python to CDK, add SAT tests. | From b98580ca5f42b4d7bfb8d96449a5ebbabc78a92c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Feb 2024 11:49:13 +0100 Subject: [PATCH 11/13] only validate spec --- .../test/steps/python_connectors.py | 24 ++---- .../test_tests/test_python_connectors.py | 84 ++++++++++--------- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 0ae61a61a877..b2d890bb0617 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -206,17 +206,14 @@ async def _run(self, connector_under_test: Container) -> StepResult: Returns: StepResult: Failure or success of the unit tests with stdout and stdout. """ + print(self.context.connector.metadata) if dpath.util.get(self.context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") test_environment = await self.install_testing_environment(with_poetry(self.context)) - connector_secrets = await self.context.get_connector_secrets() - first_secret_name = list(connector_secrets.keys())[0] test_execution = test_environment.with_( - hacks.never_fail_exec( - ["airbyte-lib-validate-source", "--connector-dir", ".", "--sample-config", "secrets/" + first_secret_name] - ) + hacks.never_fail_exec(["airbyte-lib-validate-source", "--connector-dir", ".", "--validate-install-only"]) ) return await self.get_step_result(test_execution) @@ -227,21 +224,16 @@ async def install_testing_environment( ) -> Container: """Add airbyte-lib and secrets to the test environment.""" context: ConnectorContext = self.context - secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets") container_with_test_deps = await pipelines.dagger.actions.python.common.with_python_package( self.context, built_connector_container.with_entrypoint([]), str(context.connector.code_directory) ) - return ( - container_with_test_deps.with_exec( - [ - "pip", - "install", - "airbyte-lib", - ] - ) - # Mount the secrets - .with_(secret_mounting_function).with_env_variable("PYTHONPATH", ".") + return container_with_test_deps.with_exec( + [ + "pip", + "install", + "airbyte-lib", + ] ) diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py index d271543362f3..4b36571fb7fa 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from unittest.mock import patch + import pytest from connector_ops.utils import Connector, ConnectorLanguage from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages @@ -113,32 +115,11 @@ def compatible_connector(self): return Connector("source-faker") @pytest.fixture - def valid_secrets(self, dagger_client): - return { - "valid_config.json": dagger_client.set_secret( - "faker_secret_config", - """{ - "count": 10, - "seed": 0, - "parallelism": 1, - "always_updated": false -}""", - ) - } - - @pytest.fixture - def invalid_secrets(self, dagger_client): - return { - "invalid_config.json": dagger_client.set_secret( - "faker_secret_config", - """{ - "count": "a string even though it should be a number" -}""", - ) - } + def incompatible_connector(self): + return Connector("source-postgres") @pytest.fixture - def context_for_valid_connector(self, mocker, compatible_connector, dagger_client, current_platform): + def context_for_valid_connector(self, compatible_connector, dagger_client, current_platform): context = ConnectorContext( pipeline_name="test airbyte-lib validation", connector=compatible_connector, @@ -150,27 +131,48 @@ def context_for_valid_connector(self, mocker, compatible_connector, dagger_clien targeted_platforms=[current_platform], ) context.dagger_client = dagger_client - context.get_connector_secrets = mocker.AsyncMock(return_value={}) return context @pytest.fixture - async def connector_container(self, context_for_valid_connector, current_platform): - result = await BuildConnectorImages(context_for_valid_connector).run() - return result.output_artifact[current_platform] + def context_for_invalid_connector(self, incompatible_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test airbyte-lib validation", + connector=incompatible_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context - async def test__run_validation_success(self, mocker, valid_secrets, context_for_valid_connector: ConnectorContext, connector_container): - context_for_valid_connector.get_connector_secrets = mocker.AsyncMock(return_value=valid_secrets) - result = await AirbyteLibValidation(context_for_valid_connector)._run(connector_container) + async def test__run_validation_success(self, mocker, context_for_valid_connector: ConnectorContext): + result = await AirbyteLibValidation(context_for_valid_connector)._run(mocker.MagicMock()) assert isinstance(result, StepResult) assert result.status == StepStatus.SUCCESS - assert "Creating source and validating spec and version..." in result.stdout - assert "Running check..." in result.stdout - assert "Fetching streams..." in result.stdout - assert "Trying to read from stream" in result.stdout - - async def test__run_validation_fail(self, mocker, invalid_secrets, context_for_valid_connector: ConnectorContext, connector_container): - context_for_valid_connector.get_connector_secrets = mocker.AsyncMock(return_value=invalid_secrets) - result = await AirbyteLibValidation(context_for_valid_connector)._run(connector_container) + assert "Creating source and validating spec is returned successfully..." in result.stdout + + async def test__run_validation_skip_unpublished_connector( + self, + mocker, + context_for_invalid_connector: ConnectorContext, + ): + result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock()) assert isinstance(result, StepResult) - assert result.status == StepStatus.FAILURE - assert "'a string even though it should be a number' is not of type 'integer'" in result.stderr + assert result.status == StepStatus.SKIPPED + + async def test__run_validation_fail( + self, + mocker, + context_for_invalid_connector: ConnectorContext, + ): + metadata = context_for_invalid_connector.connector.metadata + metadata["remoteRegistries"] = {"pypi": {"enabled": True, "packageName": "airbyte-source-postgres"}} + metadata_mock = mocker.PropertyMock(return_value=metadata) + with patch.object(Connector, "metadata", metadata_mock): + result = await AirbyteLibValidation(context_for_invalid_connector)._run(mocker.MagicMock()) + assert isinstance(result, StepResult) + assert result.status == StepStatus.FAILURE + assert "does not appear to be a Python project" in result.stderr From e31297d42ff52de81eacb8f8caa4bb7ef495109e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Feb 2024 17:06:21 +0100 Subject: [PATCH 12/13] remove debug --- .../airbyte_ci/connectors/test/steps/python_connectors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index b2d890bb0617..85d70de21872 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -206,7 +206,6 @@ async def _run(self, connector_under_test: Container) -> StepResult: Returns: StepResult: Failure or success of the unit tests with stdout and stdout. """ - print(self.context.connector.metadata) if dpath.util.get(self.context.connector.metadata, "remoteRegistries/pypi/enabled", default=False) is False: return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") From 86ffdbe62efb2a178753fdb93999cd72c96a0526 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 5 Feb 2024 17:07:41 +0100 Subject: [PATCH 13/13] dummy change --- .../airbyte_ci/connectors/test/steps/python_connectors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 85d70de21872..352a404a2433 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -210,7 +210,6 @@ async def _run(self, connector_under_test: Container) -> StepResult: return self.skip("Connector is not published on pypi, skipping airbyte-lib validation.") test_environment = await self.install_testing_environment(with_poetry(self.context)) - test_execution = test_environment.with_( hacks.never_fail_exec(["airbyte-lib-validate-source", "--connector-dir", ".", "--validate-install-only"]) )