airbyte-ci: Pass env vars to poetry container in test command (#34288)

airbyte-ci/connectors/pipelines/README.md

@@ -528,6 +528,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 | Option                    | Required | Default | Mapped environment variable | Description                                                                                 |
 | ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- |
 | `-c/--poetry-run-command` | True     | None    |                             | The command to run with `poetry run`                                                        |
+| `-e/--pass-env-var`       | False    | None    |                             | Host environment variable that is passed to the container running the poetry command        |
 | `--ci-requirements`       | False    |         |                             | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
 
 #### Examples
@@ -536,13 +537,17 @@ You can pass multiple `-c/--poetry-run-command` options to run multiple commands
 E.G.: running `pytest` and `mypy`:
 `airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines'`
 
+E.G.: passing the environment variable `GCP_GSM_CREDENTIALS` environment variable to the container running the poetry command:
+`airbyte-ci test airbyte-lib --pass-env-var='GCP_GSM_CREDENTIALS'`
+
 E.G.: running `pytest` on a specific test folder:
 `airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest tests/unit_tests'`
 
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 3.5.0   | [#33313](https://github.com/airbytehq/airbyte/pull/33313)  | Pass extra params after Gradle tasks.                                                                             |
 | 3.4.2   | [#34301](https://github.com/airbytehq/airbyte/pull/34301)  | Pass extra params after Gradle tasks.                                                                             |
 | 3.4.1   | [#34067](https://github.com/airbytehq/airbyte/pull/34067)  | Use dagster-cloud 1.5.7 for deploy |
 | 3.4.0   | [#34276](https://github.com/airbytehq/airbyte/pull/34276)  | Introduce `--only-step` option for connector tests.                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import logging
+import os
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -34,6 +35,13 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple
     return await container.stdout(), await container.stderr()
 
 
+def validate_env_vars_exist(_ctx: dict, _param: dict, value: List[str]) -> List[str]:
+    for var in value:
+        if var not in os.environ:
+            raise click.BadParameter(f"Environment variable {var} does not exist.")
+    return value
+
+
 @click.command()
 @click.argument("poetry_package_path")
 @click_ci_requirements_option()
@@ -44,6 +52,15 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple
     help="The poetry run command to run.",
     required=True,
 )
+@click.option(
+    "--pass-env-var",
+    "-e",
+    "passed_env_vars",
+    multiple=True,
+    help="The environment variables to pass to the container.",
+    required=False,
+    callback=validate_env_vars_exist,
+)
 @click_merge_args_into_context_obj
 @pass_pipeline_context
 @click_ignore_unused_kwargs
@@ -112,6 +129,11 @@ async def test(pipeline_context: ClickPipelineContext) -> None:
         .with_workdir(f"/airbyte/{poetry_package_path}")
     )
 
+    # register passed env vars as secrets and add them to the container
+    for var in pipeline_context.params["passed_env_vars"]:
+        secret = dagger_client.set_secret(var, os.environ[var])
+        test_container = test_container.with_secret_variable(var, secret)
+
     soon_command_executions_results = []
     async with asyncer.create_task_group() as poetry_commands_task_group:
         for command in commands_to_run:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.4.2"
+version = "3.5.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]