From 35872352bd47d5bfeddcf166f187bb55f2a1162f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 7 Aug 2023 14:01:58 +0200 Subject: [PATCH 1/5] rename tests/tests to tests/test_tests --- .../connectors/pipelines/tests/{tests => test_tests}/__init__.py | 0 .../pipelines/tests/{tests => test_tests}/test_common.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename airbyte-ci/connectors/pipelines/tests/{tests => test_tests}/__init__.py (100%) rename airbyte-ci/connectors/pipelines/tests/{tests => test_tests}/test_common.py (100%) diff --git a/airbyte-ci/connectors/pipelines/tests/tests/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py similarity index 100% rename from airbyte-ci/connectors/pipelines/tests/tests/__init__.py rename to airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py diff --git a/airbyte-ci/connectors/pipelines/tests/tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py similarity index 100% rename from airbyte-ci/connectors/pipelines/tests/tests/test_common.py rename to airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py From b1b23c88ab609676043ece3da23ecd252fb489f1 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 7 Aug 2023 14:02:55 +0200 Subject: [PATCH 2/5] user check_path_in_workdir instead of get_file_contenxt --- .../pipelines/actions/environments.py | 10 +++-- .../connectors/pipelines/pipelines/bases.py | 2 +- .../tests/test_actions/test_environments.py | 39 +++++++++++++++++++ .../pipelines/tests/test_environments.py | 3 ++ .../connectors/pipelines/tests/test_utils.py | 17 +++++++- 5 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_environments.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py index 513f3ae79161..5ee973f5e098 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py @@ -24,7 +24,7 @@ LICENSE_SHORT_FILE_PATH, PYPROJECT_TOML_FILE_PATH, ) -from pipelines.utils import get_file_contents +from pipelines.utils import check_path_in_workdir, get_file_contents if TYPE_CHECKING: from pipelines.contexts import ConnectorContext, PipelineContext @@ -312,9 +312,13 @@ async def with_installed_python_package( for dependency_directory in local_dependencies: container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) - if await get_file_contents(container, "setup.py"): + has_setup_py, has_requirements_txt = await check_path_in_workdir(container, "setup.py"), await check_path_in_workdir( + container, "requirements.txt" + ) + + if has_setup_py: container = container.with_exec(install_connector_package_cmd) - if await get_file_contents(container, "requirements.txt"): + if has_requirements_txt: container = container.with_exec(install_requirements_cmd) if additional_dependency_groups: diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py index 949852511b3e..394117d7bb0a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/bases.py +++ b/airbyte-ci/connectors/pipelines/pipelines/bases.py @@ -187,7 +187,7 @@ def should_retry(self, step_result: StepResult) -> bool: async def retry(self, step_result, *args, **kwargs) -> StepResult: self.retry_count += 1 self.logger.warn( - f"Failed with error: {step_result.stderr}. Retry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." + f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." ) await anyio.sleep(self.retry_delay.total_seconds()) return await self.run(*args, **kwargs) diff --git a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py new file mode 100644 index 000000000000..f48c061dbec2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from connector_ops.utils import Connector +from pipelines.actions import environments +from pipelines.contexts import PipelineContext + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def python_connector() -> Connector: + return Connector("source-openweather") + + +@pytest.fixture +def context(dagger_client): + context = PipelineContext( + pipeline_name="test", + is_local=True, + git_branch="test", + git_revision="test", + ) + context.dagger_client = dagger_client + return context + + +async def test_with_installed_python_package(context, python_connector): + python_environment = context.dagger_client.container().from_("python:3.9") + installed_connector_package = await environments.with_installed_python_package( + context, + python_environment, + str(python_connector.code_directory), + ) + await installed_connector_package.with_exec(["python", "main.py", "spec"]) diff --git a/airbyte-ci/connectors/pipelines/tests/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_environments.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_environments.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py index 35984a1bcef3..4816356cbee3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_utils.py @@ -5,7 +5,7 @@ from unittest import mock import pytest -from connector_ops.utils import ConnectorLanguage +from connector_ops.utils import Connector, ConnectorLanguage from pipelines import utils from tests.utils import pick_a_random_connector @@ -165,3 +165,18 @@ def test_no_modified_files_in_connector_directory(): result = utils.get_connector_modified_files(connector, all_modified_files) assert result == frozenset() + + +@pytest.mark.anyio +async def test_check_path_in_workdir(dagger_client): + connector = Connector("source-openweather") + container = ( + dagger_client.container() + .from_("bash") + .with_mounted_directory(str(connector.code_directory), dagger_client.host().directory(str(connector.code_directory))) + .with_workdir(str(connector.code_directory)) + ) + assert await utils.check_path_in_workdir(container, "metadata.yaml") + assert await utils.check_path_in_workdir(container, "setup.py") + assert await utils.check_path_in_workdir(container, "requirements.txt") + assert await utils.check_path_in_workdir(container, "not_existing_file") is False From ac2d8719829eadf2e3cedd833aa1b81d51eccfd0 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 7 Aug 2023 14:14:24 +0200 Subject: [PATCH 3/5] bump version --- airbyte-ci/connectors/pipelines/README.md | 38 +++++++++---------- .../connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index e23ea25beb46..a90e070e7d94 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -377,25 +377,25 @@ This command runs the Python tests for a airbyte-ci poetry package. `airbyte-ci tests connectors/pipelines` ## Changelog - -| Version | PR | Description | -|---------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------| -| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | -| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | -| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | -| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | -| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | -| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | -| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | -| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | -| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | -| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | -| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | -| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | -| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | -| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | -| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | -| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | +| Version | PR | Description | +| ------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | +| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | +| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | +| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | +| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | +| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | +| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | +| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | +| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | +| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | +| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | +| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | +| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | +| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | +| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | +| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | ## More info This project is owned by the Connectors Operations team. diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index b4c4f7dee61c..063bd05eb1dd 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "0.4.6" +version = "0.4.7" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 369002556ba330c0a4752f97e0ed86525cf6f15c Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 7 Aug 2023 14:15:12 +0200 Subject: [PATCH 4/5] DEMO - to revert --- airbyte-integrations/connectors/source-intercom/metadata.yaml | 1 + airbyte-integrations/connectors/source-openweather/metadata.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index d81bbb33e242..3352acc26906 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -26,3 +26,4 @@ data: ql: 300 supportLevel: certified metadataSpecVersion: "1.0" +# TODO revert \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index 2adda15ad340..238121118d52 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -22,3 +22,4 @@ data: ql: 200 supportLevel: community metadataSpecVersion: "1.0" +# TODO: revert \ No newline at end of file From b1f95647504a11ba163003d4e202b766aae25f6a Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 7 Aug 2023 18:21:50 +0200 Subject: [PATCH 5/5] Revert "DEMO - to revert" This reverts commit 369002556ba330c0a4752f97e0ed86525cf6f15c. --- airbyte-integrations/connectors/source-intercom/metadata.yaml | 1 - airbyte-integrations/connectors/source-openweather/metadata.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index 3352acc26906..d81bbb33e242 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -26,4 +26,3 @@ data: ql: 300 supportLevel: certified metadataSpecVersion: "1.0" -# TODO revert \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index 238121118d52..2adda15ad340 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -22,4 +22,3 @@ data: ql: 200 supportLevel: community metadataSpecVersion: "1.0" -# TODO: revert \ No newline at end of file