From a76f8c327fd4d5a75f678b704cc8d0fd102996a1 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 26 Mar 2024 18:45:24 -0500 Subject: [PATCH 1/2] airbyte ci test to support --extras --- .github/workflows/airbyte-ci-tests.yml | 1 + airbyte-ci/connectors/base_images/pyproject.toml | 2 +- airbyte-ci/connectors/ci_credentials/pyproject.toml | 2 +- airbyte-ci/connectors/common_utils/pyproject.toml | 2 +- airbyte-ci/connectors/connector_ops/pyproject.toml | 2 +- airbyte-ci/connectors/connectors_qa/pyproject.toml | 2 +- airbyte-ci/connectors/metadata_service/lib/pyproject.toml | 2 +- .../connectors/metadata_service/orchestrator/pyproject.toml | 2 +- .../pipelines/pipelines/airbyte_ci/test/models.py | 3 ++- .../pipelines/pipelines/airbyte_ci/test/pipeline.py | 6 +++++- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- airbyte-lib/pyproject.toml | 2 +- 12 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index 37240d132791..bf24d3179068 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -30,6 +30,7 @@ jobs: with: # Note: expressions within a filter are OR'ed filters: | + # This list is duplicated in `pipelines/airbyte_ci/test/__init__.py` internal_poetry_packages: - airbyte-lib/** - airbyte-ci/connectors/pipelines/** diff --git a/airbyte-ci/connectors/base_images/pyproject.toml b/airbyte-ci/connectors/base_images/pyproject.toml index d853b591b277..3630f4f3e8e0 100644 --- a/airbyte-ci/connectors/base_images/pyproject.toml +++ b/airbyte-ci/connectors/base_images/pyproject.toml @@ -36,6 +36,6 @@ publish = "base_images.commands:publish_existing_version" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] mount_docker_socket = true diff --git a/airbyte-ci/connectors/ci_credentials/pyproject.toml b/airbyte-ci/connectors/ci_credentials/pyproject.toml index a85705db5762..efafda74980f 100644 --- a/airbyte-ci/connectors/ci_credentials/pyproject.toml +++ b/airbyte-ci/connectors/ci_credentials/pyproject.toml @@ -32,5 +32,5 @@ ci_credentials = "ci_credentials.main:ci_credentials" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] diff --git a/airbyte-ci/connectors/common_utils/pyproject.toml b/airbyte-ci/connectors/common_utils/pyproject.toml index ab21cfab1844..2958161c4573 100644 --- a/airbyte-ci/connectors/common_utils/pyproject.toml +++ b/airbyte-ci/connectors/common_utils/pyproject.toml @@ -27,6 +27,6 @@ build-backend = "poetry.core.masonry.api" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] # Disable poe tasks as tests are not passing ATM poe_tasks = [] diff --git a/airbyte-ci/connectors/connector_ops/pyproject.toml b/airbyte-ci/connectors/connector_ops/pyproject.toml index c3037e9230ef..0d06fa0373e0 100644 --- a/airbyte-ci/connectors/connector_ops/pyproject.toml +++ b/airbyte-ci/connectors/connector_ops/pyproject.toml @@ -38,5 +38,5 @@ allowed-hosts-checks = "connector_ops.allowed_hosts_checks:check_allowed_hosts" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] diff --git a/airbyte-ci/connectors/connectors_qa/pyproject.toml b/airbyte-ci/connectors/connectors_qa/pyproject.toml index e1b6c5f98f6d..ff2bd5be41e5 100644 --- a/airbyte-ci/connectors/connectors_qa/pyproject.toml +++ b/airbyte-ci/connectors/connectors_qa/pyproject.toml @@ -40,6 +40,6 @@ type_check = "mypy src --disallow-untyped-defs" lint = "ruff check src" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["type_check", "lint", "test"] required_environment_variables = ["DOCKER_HUB_USERNAME", "DOCKER_HUB_PASSWORD",] diff --git a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml index b4f0389e433b..338d76479d20 100644 --- a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml @@ -36,7 +36,7 @@ promote-connector-to-latest = "gsutil -m rsync -r -d gs://$TARGET_BUCKET/metada test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] [build-system] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml index 1cd1f9abfa41..c58e59fc292a 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml @@ -49,5 +49,5 @@ module_name = "orchestrator" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py index fd4ed95de5ab..d1ba68f4be03 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py @@ -11,7 +11,8 @@ class AirbyteCiPackageConfiguration(BaseModel): required_environment_variables: Set[str] = Field( set(), description="List of unique required environment variables to pass to the container running the poe task" ) - extra_poetry_groups: Set[str] = Field(set(), description="List of unique extra poetry groups to install") + poetry_extras: Set[str] = Field(set(), description="List of unique poetry extras to install") + optional_poetry_groups: Set[str] = Field(set(), description="List of unique poetry groups to install") side_car_docker_engine: bool = Field( False, description="Flag indicating the use of a sidecar Docker engine during the poe task executions" ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py index f8ada153f44c..ab99ecff4919 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py @@ -191,7 +191,11 @@ def prepare_container_for_poe_tasks( container = container.with_workdir(f"/airbyte/{poetry_package_path}") # Install the poetry package - container = container.with_exec(["poetry", "install"] + [f"--with={group}" for group in airbyte_ci_package_config.extra_poetry_groups]) + container = container.with_exec( + ["poetry", "install"] + + [f"--with={group}" for group in airbyte_ci_package_config.optional_poetry_groups] + + [f"--extras={extra}" for extra in airbyte_ci_package_config.poetry_extras] + ) return container diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index c71f38efd518..e5fc5c2460f6 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -62,6 +62,6 @@ type_check = "mypy pipelines --disallow-untyped-defs" lint = "ruff check pipelines" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["type_check", "lint", "test"] mount_docker_socket = true diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml index 8634bf4d1f3b..da71cae42b29 100644 --- a/airbyte-lib/pyproject.toml +++ b/airbyte-lib/pyproject.toml @@ -255,7 +255,7 @@ airbyte-lib-validate-source = "airbyte_lib.validate:run" test = "pytest tests" [tool.airbyte_ci] -extra_poetry_groups = ["dev"] +optional_poetry_groups = ["dev"] poe_tasks = ["test"] required_environment_variables = ["GCP_GSM_CREDENTIALS"] side_car_docker_engine = true From 695f18e7fe11217eff4d40f6ee9675dd3d8af7ab Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 26 Mar 2024 23:15:00 -0500 Subject: [PATCH 2/2] add changelog entry --- airbyte-ci/connectors/pipelines/README.md | 1 + airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 334be5f9dfcd..42b356ea7aea 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -652,6 +652,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| 4.6.3 | [#36527](https://github.com/airbytehq/airbyte/pull/36527) | Handle extras as well as groups in `airbyte ci test` [poetry packages] | | 4.6.2 | [#36220](https://github.com/airbytehq/airbyte/pull/36220) | Allow using `migrate-to-base-image` without PULL_REQUEST_NUMBER | | 4.6.1 | [#36319](https://github.com/airbytehq/airbyte/pull/36319) | Fix `ValueError` related to PR number in migrate-to-poetry | | 4.6.0 | [#35583](https://github.com/airbytehq/airbyte/pull/35583) | Implement the `airbyte-ci connectors migrate-to-poetry` command. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index e5fc5c2460f6..11ae146ea88c 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.6.2" +version = "4.6.3" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]