From f3503ae876cf59afdc742fdf6d0303ad7b378d48 Mon Sep 17 00:00:00 2001 From: Augustin Date: Mon, 15 Jan 2024 19:09:33 +0100 Subject: [PATCH] airbyte-ci: CLI exposes CI requirements (#34218) --- airbyte-ci/connectors/pipelines/README.md | 48 +++++++++++-------- .../airbyte_ci/connectors/publish/commands.py | 2 + .../airbyte_ci/connectors/test/commands.py | 2 + .../pipelines/airbyte_ci/format/commands.py | 3 +- .../pipelines/airbyte_ci/metadata/commands.py | 2 + .../pipelines/airbyte_ci/test/commands.py | 3 +- .../pipelines/pipelines/cli/airbyte_ci.py | 10 +++- .../pipelines/cli/click_decorators.py | 31 +++++++++++- .../pipelines/models/ci_requirements.py | 33 +++++++++++++ .../connectors/pipelines/pyproject.toml | 2 +- 10 files changed, 110 insertions(+), 26 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 552e103f2bdb..20e7bdf6ad3e 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -258,13 +258,14 @@ flowchart TD #### Options -| Option | Multiple | Default value | Description | -| ------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--skip-step/-x` | True | | Skip steps by id e.g. `-x unit -x acceptance` | -| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | -| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. | -| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. | -| `--.=` | True | | You can pass extra parameters for specific test steps. More details in the extra parameters section below | +| Option | Multiple | Default value | Description | +| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--skip-step/-x` | True | | Skip steps by id e.g. `-x unit -x acceptance` | +| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | +| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. | +| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. | +| `--.=` | True | | You can pass extra parameters for specific test steps. More details in the extra parameters section below | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. Note: @@ -370,6 +371,8 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod | `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | | `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | | `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | + I've added an empty "Default" column, and you can fill in the default values as needed. @@ -462,9 +465,10 @@ Available commands: ### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------ | -------- | ------- | --------------------------- | ---------------------------------------------- | -| `--quiet/-q` | False | False | | Hide formatter execution details in reporting. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `--quiet/-q` | False | False | | Hide formatter execution details in reporting. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | ### Examples @@ -517,9 +521,10 @@ This command runs the Python tests for a airbyte-ci poetry package. #### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------ | -| `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | #### Examples You can pass multiple `-c/--poetry-run-command` options to run multiple commands. @@ -534,14 +539,15 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | -| 3.2.0 | [#34050](https://github.com/airbytehq/airbyte/pull/34050) | Connector test steps can take extra parameters | -| 3.1.3 | [#34136](https://github.com/airbytehq/airbyte/pull/34136) | Fix issue where dagger excludes were not being properly applied | -| 3.1.2 | [#33972](https://github.com/airbytehq/airbyte/pull/33972) | Remove secrets scrubbing hack for --is-local and other small tweaks. | -| 3.1.1 | [#33979](https://github.com/airbytehq/airbyte/pull/33979) | Fix AssertionError on report existence again | -| 3.1.0 | [#33994](https://github.com/airbytehq/airbyte/pull/33994) | Log more context information in CI. | -| 3.0.2 | [#33987](https://github.com/airbytehq/airbyte/pull/33987) | Fix type checking issue when running --help | -| 3.0.1 | [#33981](https://github.com/airbytehq/airbyte/pull/33981) | Fix issues with deploying dagster, pin pendulum version in dagster-cli install | -| 3.0.0 | [#33582](https://github.com/airbytehq/airbyte/pull/33582) | Upgrade to Dagger 0.9.5 | +| 3.3.0 | [#34218](https://github.com/airbytehq/airbyte/pull/34218) | Introduce `--ci-requirements` option for client defined CI runners. | +| 3.2.0 | [#34050](https://github.com/airbytehq/airbyte/pull/34050) | Connector test steps can take extra parameters | +| 3.1.3 | [#34136](https://github.com/airbytehq/airbyte/pull/34136) | Fix issue where dagger excludes were not being properly applied | +| 3.1.2 | [#33972](https://github.com/airbytehq/airbyte/pull/33972) | Remove secrets scrubbing hack for --is-local and other small tweaks. | +| 3.1.1 | [#33979](https://github.com/airbytehq/airbyte/pull/33979) | Fix AssertionError on report existence again | +| 3.1.0 | [#33994](https://github.com/airbytehq/airbyte/pull/33994) | Log more context information in CI. | +| 3.0.2 | [#33987](https://github.com/airbytehq/airbyte/pull/33987) | Fix type checking issue when running --help | +| 3.0.1 | [#33981](https://github.com/airbytehq/airbyte/pull/33981) | Fix issues with deploying dagster, pin pendulum version in dagster-cli install | +| 3.0.0 | [#33582](https://github.com/airbytehq/airbyte/pull/33582) | Upgrade to Dagger 0.9.5 | | 2.14.3 | [#33964](https://github.com/airbytehq/airbyte/pull/33964) | Reintroduce mypy with fixes for AssertionError on publish and missing report URL on connector test commit status. | | 2.14.2 | [#33954](https://github.com/airbytehq/airbyte/pull/33954) | Revert mypy changes | | 2.14.1 | [#33956](https://github.com/airbytehq/airbyte/pull/33956) | Exclude pnpm lock files from auto-formatting | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index e57b930304b9..0de1d7a2032f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -8,6 +8,7 @@ from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.publish.pipeline import reorder_contexts, run_connector_publish_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.consts import ContextState @@ -15,6 +16,7 @@ @click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.") +@click_ci_requirements_option() @click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool) @click.option( "--spec-cache-gcs-credentials", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 07f48bc5ca75..9d97f0c90a7b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -11,6 +11,7 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState from pipelines.helpers.execution import argument_parsing @@ -27,6 +28,7 @@ ignore_unknown_options=True, ), ) +@click_ci_requirements_option() @click.option( "--code-tests-only", is_flag=True, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 59e84e10c4aa..a1f2d3cc613f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -14,7 +14,7 @@ import asyncclick as click from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter from pipelines.airbyte_ci.format.format_command import FormatCommand -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import StepStatus @@ -25,6 +25,7 @@ help="Commands related to formatting.", ) @click.option("--quiet", "-q", help="Hide details of the formatter execution.", default=False, is_flag=True) +@click_ci_requirements_option() @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py index 302da3b11c1b..ca856d9bbb67 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py @@ -3,12 +3,14 @@ # import asyncclick as click +from pipelines.cli.click_decorators import click_ci_requirements_option from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand # MAIN GROUP @click.group(help="Commands related to the metadata service.") +@click_ci_requirements_option() @click.pass_context def metadata(ctx: click.Context) -> None: pass diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 0ea67650eb19..7bf140211e78 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -9,7 +9,7 @@ import asyncclick as click import asyncer -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -36,6 +36,7 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple @click.command() @click.argument("poetry_package_path") +@click_ci_requirements_option() @click.option( "-c", "--poetry-run-command", diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index c7a463a160c9..8779fee5eab1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -24,7 +24,12 @@ from github import PullRequest from pipelines import main_logger from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag -from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import ( + CI_REQUIREMENTS_OPTION_NAME, + click_append_to_context_object, + click_ignore_unused_kwargs, + click_merge_args_into_context_obj, +) from pipelines.cli.confirm_prompt import pre_confirm_all_flag from pipelines.cli.lazy_group import LazyGroup from pipelines.cli.telemetry import click_track_command @@ -83,6 +88,9 @@ def check_local_docker_configuration() -> None: def is_dagger_run_enabled_by_default() -> bool: + if CI_REQUIREMENTS_OPTION_NAME in sys.argv: + return False + dagger_run_by_default = [ ["connectors", "test"], ["connectors", "build"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 47bd4ccb243a..b88f582c6e37 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -5,9 +5,14 @@ import functools import inspect from functools import wraps -from typing import Any, Callable, Type +from typing import Any, Callable, Type, TypeVar import asyncclick as click +from pipelines.models.ci_requirements import CIRequirements + +_AnyCallable = Callable[..., Any] +FC = TypeVar("FC", bound="_AnyCallable | click.core.Command") +CI_REQUIREMENTS_OPTION_NAME = "--ci-requirements" def _contains_var_kwarg(f: Callable) -> bool: @@ -121,3 +126,27 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 return f(*args, **kwargs) return decorated_function + + +def click_ci_requirements_option() -> Callable[[FC], FC]: + """Add a --ci-requirements option to the command. + + Returns: + Callable[[FC], FC]: The decorated command. + """ + + def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None: + if value: + ci_requirements = CIRequirements() + click.echo(ci_requirements.to_json()) + ctx.exit() + + return click.decorators.option( + CI_REQUIREMENTS_OPTION_NAME, + is_flag=True, + expose_value=False, + is_eager=True, + flag_value=True, + help="Show the CI requirements and exit. It used to make airbyte-ci client define the CI runners it will run on.", + callback=callback, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py new file mode 100644 index 000000000000..7eb8a9157b57 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from dataclasses import dataclass +from importlib import metadata + +INFRA_SUPPORTED_DAGGER_VERSIONS = { + "0.6.4", + "0.9.5", +} + + +@dataclass +class CIRequirements: + """ + A dataclass to store the CI requirements. + It used to make airbyte-ci client define the CI runners it will run on. + """ + + dagger_version = metadata.version("dagger-io") + + def __post_init__(self) -> None: + if self.dagger_version not in INFRA_SUPPORTED_DAGGER_VERSIONS: + raise ValueError( + f"Unsupported dagger version: {self.dagger_version}. " f"Supported versions are: {INFRA_SUPPORTED_DAGGER_VERSIONS}." + ) + + def to_json(self) -> str: + return json.dumps( + { + "dagger_version": self.dagger_version, + } + ) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index ef811f62aaca..4c7be71da341 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.2.0" +version = "3.3.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]