airbyte-ci: CLI exposes CI requirements (#34218)

airbytehq · Jan 15, 2024 · f3503ae · f3503ae
1 parent 29852cf
commit f3503ae
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 26 deletions.
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -258,13 +258,14 @@ flowchart TD
 
 #### Options
 
-| Option              | Multiple | Default value | Description                                                                                                                                                                                              |
-| ------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--skip-step/-x`    | True     |               | Skip steps by id e.g. `-x unit -x acceptance`                                                                                                                                                            |
-| `--fail-fast`       | False    | False         | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result.                      |
-| `--code-tests-only` | True     | False         | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. |
-| `--concurrent-cat`  | False    | False         | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits.                                                                                              |
-| `--<step-id>.<extra-parameter>=<extra-parameter-value>`  | True    |          | You can pass extra parameters for specific test steps. More details in the extra parameters section below                                                                                              |
+| Option                                                  | Multiple | Default value | Description                                                                                                                                                                                              |
+| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--skip-step/-x`                                        | True     |               | Skip steps by id e.g. `-x unit -x acceptance`                                                                                                                                                            |
+| `--fail-fast`                                           | False    | False         | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result.                      |
+| `--code-tests-only`                                     | True     | False         | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. |
+| `--concurrent-cat`                                      | False    | False         | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits.                                                                                              |
+| `--<step-id>.<extra-parameter>=<extra-parameter-value>` | True     |               | You can pass extra parameters for specific test steps. More details in the extra parameters section below                                                                                                |
+| `--ci-requirements`                                     | False    |               |                                                                                                                                                                                                          | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. 
 
 Note:
 
@@ -370,6 +371,8 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod
 | `--metadata-service-bucket-name`     | False    |                 | `METADATA_SERVICE_BUCKET_NAME`     | The name of the GCS bucket where metadata files will be uploaded.                                                                                                                         |
 | `--slack-webhook`                    | False    |                 | `SLACK_WEBHOOK`                    | The Slack webhook URL to send notifications to.                                                                                                                                           |
 | `--slack-channel`                    | False    |                 | `SLACK_CHANNEL`                    | The Slack channel name to send notifications to.                                                                                                                                          |
+| `--ci-requirements`                  | False    |                 |                                    | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use.                                                                                               |
+
 
 I've added an empty "Default" column, and you can fill in the default values as needed.
 
@@ -462,9 +465,10 @@ Available commands:
 
 ### Options
 
-| Option       | Required | Default | Mapped environment variable | Description                                    |
-| ------------ | -------- | ------- | --------------------------- | ---------------------------------------------- |
-| `--quiet/-q` | False    | False   |                             | Hide formatter execution details in reporting. |
+| Option              | Required | Default | Mapped environment variable | Description                                                                                 |
+| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- |
+| `--quiet/-q`        | False    | False   |                             | Hide formatter execution details in reporting.                                              |
+| `--ci-requirements` | False    |         |                             | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
 
 ### Examples
 
@@ -517,9 +521,10 @@ This command runs the Python tests for a airbyte-ci poetry package.
 
 #### Options
 
-| Option                    | Required | Default | Mapped environment variable | Description                          |
-| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------ |
-| `-c/--poetry-run-command` | True     | None    |                             | The command to run with `poetry run` |
+| Option                    | Required | Default | Mapped environment variable | Description                                                                                 |
+| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- |
+| `-c/--poetry-run-command` | True     | None    |                             | The command to run with `poetry run`                                                        |
+| `--ci-requirements`       | False    |         |                             | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
 
 #### Examples
 You can pass multiple `-c/--poetry-run-command` options to run multiple commands.
@@ -534,14 +539,15 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
-| 3.2.0  | [#34050](https://github.com/airbytehq/airbyte/pull/34050)  | Connector test steps can take extra parameters |
-| 3.1.3   | [#34136](https://github.com/airbytehq/airbyte/pull/34136)  | Fix issue where dagger excludes were not being properly applied |
-| 3.1.2   | [#33972](https://github.com/airbytehq/airbyte/pull/33972)  | Remove secrets scrubbing hack for --is-local and other small tweaks. |
-| 3.1.1   | [#33979](https://github.com/airbytehq/airbyte/pull/33979)  | Fix AssertionError on report existence again |
-| 3.1.0   | [#33994](https://github.com/airbytehq/airbyte/pull/33994)  | Log more context information in CI. |
-| 3.0.2   | [#33987](https://github.com/airbytehq/airbyte/pull/33987)  | Fix type checking issue when running --help |
-| 3.0.1   | [#33981](https://github.com/airbytehq/airbyte/pull/33981)  | Fix issues with deploying dagster, pin pendulum version in dagster-cli install |
-| 3.0.0   | [#33582](https://github.com/airbytehq/airbyte/pull/33582)  | Upgrade to Dagger 0.9.5 |
+| 3.3.0   | [#34218](https://github.com/airbytehq/airbyte/pull/34218)  | Introduce `--ci-requirements` option for client defined CI runners.                                               |
+| 3.2.0   | [#34050](https://github.com/airbytehq/airbyte/pull/34050)  | Connector test steps can take extra parameters                                                                    |
+| 3.1.3   | [#34136](https://github.com/airbytehq/airbyte/pull/34136)  | Fix issue where dagger excludes were not being properly applied                                                   |
+| 3.1.2   | [#33972](https://github.com/airbytehq/airbyte/pull/33972)  | Remove secrets scrubbing hack for --is-local and other small tweaks.                                              |
+| 3.1.1   | [#33979](https://github.com/airbytehq/airbyte/pull/33979)  | Fix AssertionError on report existence again                                                                      |
+| 3.1.0   | [#33994](https://github.com/airbytehq/airbyte/pull/33994)  | Log more context information in CI.                                                                               |
+| 3.0.2   | [#33987](https://github.com/airbytehq/airbyte/pull/33987)  | Fix type checking issue when running --help                                                                       |
+| 3.0.1   | [#33981](https://github.com/airbytehq/airbyte/pull/33981)  | Fix issues with deploying dagster, pin pendulum version in dagster-cli install                                    |
+| 3.0.0   | [#33582](https://github.com/airbytehq/airbyte/pull/33582)  | Upgrade to Dagger 0.9.5                                                                                           |
 | 2.14.3  | [#33964](https://github.com/airbytehq/airbyte/pull/33964)  | Reintroduce mypy with fixes for AssertionError on publish and missing report URL on connector test commit status. |
 | 2.14.2  | [#33954](https://github.com/airbytehq/airbyte/pull/33954)  | Revert mypy changes                                                                                               |
 | 2.14.1  | [#33956](https://github.com/airbytehq/airbyte/pull/33956)  | Exclude pnpm lock files from auto-formatting                                                                      |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py
@@ -8,13 +8,15 @@
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
 from pipelines.airbyte_ci.connectors.publish.pipeline import reorder_contexts, run_connector_publish_pipeline
+from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.confirm_prompt import confirm
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import ContextState
 from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
 
 
 @click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.")
+@click_ci_requirements_option()
 @click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool)
 @click.option(
     "--spec-cache-gcs-credentials",

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py
@@ -11,6 +11,7 @@
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline
+from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState
 from pipelines.helpers.execution import argument_parsing
@@ -27,6 +28,7 @@
         ignore_unknown_options=True,
     ),
 )
+@click_ci_requirements_option()
 @click.option(
     "--code-tests-only",
     is_flag=True,

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py
@@ -14,7 +14,7 @@
 import asyncclick as click
 from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter
 from pipelines.airbyte_ci.format.format_command import FormatCommand
-from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
 from pipelines.models.steps import StepStatus
@@ -25,6 +25,7 @@
     help="Commands related to formatting.",
 )
 @click.option("--quiet", "-q", help="Hide details of the formatter execution.", default=False, is_flag=True)
+@click_ci_requirements_option()
 @click_merge_args_into_context_obj
 @pass_pipeline_context
 @click_ignore_unused_kwargs

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py
@@ -3,12 +3,14 @@
 #
 
 import asyncclick as click
+from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 
 # MAIN GROUP
 
 
 @click.group(help="Commands related to the metadata service.")
+@click_ci_requirements_option()
 @click.pass_context
 def metadata(ctx: click.Context) -> None:
     pass

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py
@@ -9,7 +9,7 @@
 
 import asyncclick as click
 import asyncer
-from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.consts import DOCKER_VERSION
 from pipelines.helpers.utils import sh_dash_c
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
@@ -36,6 +36,7 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple
 
 @click.command()
 @click.argument("poetry_package_path")
+@click_ci_requirements_option()
 @click.option(
     "-c",
     "--poetry-run-command",

diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py
@@ -24,7 +24,12 @@
 from github import PullRequest
 from pipelines import main_logger
 from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag
-from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import (
+    CI_REQUIREMENTS_OPTION_NAME,
+    click_append_to_context_object,
+    click_ignore_unused_kwargs,
+    click_merge_args_into_context_obj,
+)
 from pipelines.cli.confirm_prompt import pre_confirm_all_flag
 from pipelines.cli.lazy_group import LazyGroup
 from pipelines.cli.telemetry import click_track_command
@@ -83,6 +88,9 @@ def check_local_docker_configuration() -> None:
 
 
 def is_dagger_run_enabled_by_default() -> bool:
+    if CI_REQUIREMENTS_OPTION_NAME in sys.argv:
+        return False
+
     dagger_run_by_default = [
         ["connectors", "test"],
         ["connectors", "build"],

diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py
@@ -5,9 +5,14 @@
 import functools
 import inspect
 from functools import wraps
-from typing import Any, Callable, Type
+from typing import Any, Callable, Type, TypeVar
 
 import asyncclick as click
+from pipelines.models.ci_requirements import CIRequirements
+
+_AnyCallable = Callable[..., Any]
+FC = TypeVar("FC", bound="_AnyCallable | click.core.Command")
+CI_REQUIREMENTS_OPTION_NAME = "--ci-requirements"
 
 
 def _contains_var_kwarg(f: Callable) -> bool:
@@ -121,3 +126,27 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any:  # noqa: ANN401
             return f(*args, **kwargs)
 
         return decorated_function
+
+
+def click_ci_requirements_option() -> Callable[[FC], FC]:
+    """Add a --ci-requirements option to the command.
+
+    Returns:
+        Callable[[FC], FC]: The decorated command.
+    """
+
+    def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None:
+        if value:
+            ci_requirements = CIRequirements()
+            click.echo(ci_requirements.to_json())
+            ctx.exit()
+
+    return click.decorators.option(
+        CI_REQUIREMENTS_OPTION_NAME,
+        is_flag=True,
+        expose_value=False,
+        is_eager=True,
+        flag_value=True,
+        help="Show the CI requirements and exit. It used to make airbyte-ci client define the CI runners it will run on.",
+        callback=callback,
+    )
diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import json
+from dataclasses import dataclass
+from importlib import metadata
+
+INFRA_SUPPORTED_DAGGER_VERSIONS = {
+    "0.6.4",
+    "0.9.5",
+}
+
+
+@dataclass
+class CIRequirements:
+    """
+    A dataclass to store the CI requirements.
+    It used to make airbyte-ci client define the CI runners it will run on.
+    """
+
+    dagger_version = metadata.version("dagger-io")
+
+    def __post_init__(self) -> None:
+        if self.dagger_version not in INFRA_SUPPORTED_DAGGER_VERSIONS:
+            raise ValueError(
+                f"Unsupported dagger version: {self.dagger_version}. " f"Supported versions are: {INFRA_SUPPORTED_DAGGER_VERSIONS}."
+            )
+
+    def to_json(self) -> str:
+        return json.dumps(
+            {
+                "dagger_version": self.dagger_version,
+            }
+        )
diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.2.0"
+version = "3.3.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]