diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 4c8a85289eaa..ab36fa02335d 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -528,6 +528,7 @@ This command runs the Python tests for a airbyte-ci poetry package. | Option | Required | Default | Mapped environment variable | Description | | ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | | `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` | +| `-e/--pass-env-var` | False | None | | Host environment variable that is passed to the container running the poetry command | | `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | #### Examples @@ -536,6 +537,9 @@ You can pass multiple `-c/--poetry-run-command` options to run multiple commands E.G.: running `pytest` and `mypy`: `airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines'` +E.G.: passing the environment variable `GCP_GSM_CREDENTIALS` environment variable to the container running the poetry command: +`airbyte-ci test airbyte-lib --pass-env-var='GCP_GSM_CREDENTIALS'` + E.G.: running `pytest` on a specific test folder: `airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest tests/unit_tests'` @@ -543,6 +547,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 3.5.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Pass extra params after Gradle tasks. | | 3.4.2 | [#34301](https://github.com/airbytehq/airbyte/pull/34301) | Pass extra params after Gradle tasks. | | 3.4.1 | [#34067](https://github.com/airbytehq/airbyte/pull/34067) | Use dagster-cloud 1.5.7 for deploy | | 3.4.0 | [#34276](https://github.com/airbytehq/airbyte/pull/34276) | Introduce `--only-step` option for connector tests. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 7bf140211e78..4dc24ae0a197 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +import os from pathlib import Path from typing import TYPE_CHECKING @@ -34,6 +35,13 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple return await container.stdout(), await container.stderr() +def validate_env_vars_exist(_ctx: dict, _param: dict, value: List[str]) -> List[str]: + for var in value: + if var not in os.environ: + raise click.BadParameter(f"Environment variable {var} does not exist.") + return value + + @click.command() @click.argument("poetry_package_path") @click_ci_requirements_option() @@ -44,6 +52,15 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple help="The poetry run command to run.", required=True, ) +@click.option( + "--pass-env-var", + "-e", + "passed_env_vars", + multiple=True, + help="The environment variables to pass to the container.", + required=False, + callback=validate_env_vars_exist, +) @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -112,6 +129,11 @@ async def test(pipeline_context: ClickPipelineContext) -> None: .with_workdir(f"/airbyte/{poetry_package_path}") ) + # register passed env vars as secrets and add them to the container + for var in pipeline_context.params["passed_env_vars"]: + secret = dagger_client.set_secret(var, os.environ[var]) + test_container = test_container.with_secret_variable(var, secret) + soon_command_executions_results = [] async with asyncer.create_task_group() as poetry_commands_task_group: for command in commands_to_run: diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 79f37a9efd60..31dba6ae56ca 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.4.2" +version = "3.5.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]