From a4994f485aef073176893862b01ec53e904d7004 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 17:42:42 -0700 Subject: [PATCH 001/141] Add lazy decorator --- .../pipelines/pipelines/cli/airbyte_ci.py | 12 +++++++++ .../pipelines/pipelines/cli/lazy_decorator.py | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 7da72b02ee66..683f8b25a0e4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -32,6 +32,17 @@ __installed_version__ = importlib.metadata.version("pipelines") +def display_welcome_message() -> None: + print(''' + █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ + ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ + ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ + ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ + ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ + ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ + ''') + + def check_up_to_date() -> bool: """Check if the installed version of pipelines is up to date.""" latest_version = get_latest_version() @@ -193,6 +204,7 @@ def airbyte_ci( ci_job_key: str, show_dagger_logs: bool, ): # noqa D103 + display_welcome_message() ctx.ensure_object(dict) check_up_to_date() ctx.obj["is_local"] = is_local diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py new file mode 100644 index 000000000000..b65e610c5212 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py @@ -0,0 +1,25 @@ +from functools import wraps +from typing import Any, Callable, Type + + +class LazyPassDecorator: + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + @wraps(f) + def decorated_function(*args: Any, **kwargs: Any) -> Any: + # Check if the kwargs already contain the arguments being passed by the decorator + decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} + # Create an instance of the class + instance = self.cls(*self.args, **decorator_kwargs) + # If function has **kwargs, we can put the instance there + if 'kwargs' in kwargs: + kwargs['kwargs'] = instance + # Otherwise, add it to positional arguments + else: + args = (*args, instance) + return f(*args, **kwargs) + return decorated_function From ccddd26128eec4542c7d5d8cb0375c5286b0ad98 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 18:13:28 -0700 Subject: [PATCH 002/141] Add playground subcommands --- .../pipelines/pipelines/airbyte_ci/playground/__init__.py | 3 +++ .../pipelines/pipelines/airbyte_ci/playground/commands.py | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# From 96643f192b4659d9c0fc9d33e5cd9cbe48cdad18 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 18:41:46 -0700 Subject: [PATCH 003/141] steal --- .../airbyte_ci/playground/commands.py | 32 +++ .../pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/pipelines/stolen/__init__.py | 3 + .../pipelines/pipelines/stolen/base.py | 69 ++++++ .../{cli => stolen}/lazy_decorator.py | 0 .../pipelines/pipelines/stolen/settings.py | 228 ++++++++++++++++++ .../pipelines/pipelines/stolen/singleton.py | 22 ++ 7 files changed, 355 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/base.py rename airbyte-ci/connectors/pipelines/pipelines/{cli => stolen}/lazy_decorator.py (100%) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index c941b3045795..ccff9988add5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -1,3 +1,35 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import click +from pipelines.stolen.base import PipelineContext +from pipelines.stolen.lazy_decorator import LazyPassDecorator + +from pipelines.stolen.settings import GlobalSettings + +# Stolen +settings = GlobalSettings() +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(PipelineContext, global_settings=settings) +pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) + +# NEW + +@click.command() +@click.argument("arg1") +@click.option("--opt", default="default_value") +@pass_pipeline_context +@pass_global_settings +def playground( + ctx, + arg1: str, + opt: str, +): + """Runs the tests for the given airbyte-ci package. + + Args: + poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. + test_directory (str): The directory containing the tests to run. + """ + print(f"playground: {arg1} {opt}") + print(f"ctx: {dir(ctx)}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 683f8b25a0e4..e1b55fefa8ab 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -157,6 +157,7 @@ def get_modified_files( "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", + "playground": "pipelines.airbyte_ci.playground.commands.playground", }, ) @click.version_option(__installed_version__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py new file mode 100644 index 000000000000..40758a096587 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -0,0 +1,69 @@ +import sys +from typing import Any, Callable, Optional, Union + +import dagger + +# TODO ben set up for async +# from asyncclick import Context, get_current_context + +from click import Context, get_current_context +from dagger.api.gen import Client, Container +from pydantic import BaseModel, Field, PrivateAttr + +from .settings import GlobalSettings +from .singleton import Singleton + + +# this is a bit of a hack to get around how prefect resolves parameters +# basically without this, prefect will attempt to access the context +# before we create it in main.py in order to resolve it as a parameter +# wrapping it in a function like this prevents that from happening +def get_context() -> Context: + return get_current_context() + +class PipelineContext(BaseModel, Singleton): + global_settings: GlobalSettings + dockerd_service: Optional[Container] = Field(default=None) + _dagger_client: Optional[Client] = PrivateAttr(default=None) + _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + + class Config: + arbitrary_types_allowed=True + + def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): + """ + Initialize the PipelineContext instance. + + This method checks the _initialized flag for the PipelineContext class in the Singleton base class. + If the flag is False, the initialization logic is executed and the flag is set to True. + If the flag is True, the initialization logic is skipped. + + This ensures that the initialization logic is only executed once, even if the PipelineContext instance is retrieved multiple times. + This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). + """ + if not Singleton._initialized[PipelineContext]: + super().__init__(global_settings=global_settings, **data) + self.set_global_prefect_tag_context() + Singleton._initialized[PipelineContext] = True + + import asyncio + + _dagger_client_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock) + + async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name: Optional[str] = None) -> Client: + if not self._dagger_client: + async with self._dagger_client_lock: + if not self._dagger_client: + connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) + self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + client = self._dagger_client + assert client, "Error initializing Dagger client" + return client.pipeline(pipeline_name) if pipeline_name else client + + +class GlobalContext(BaseModel, Singleton): + pipeline_context: Optional[PipelineContext] = Field(default=None) + click_context: Optional[Context] = Field(default=None) + + class Config: + arbitrary_types_allowed = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py rename to airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py new file mode 100644 index 000000000000..16a20542c171 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py @@ -0,0 +1,228 @@ +import os +import platform +from typing import Any, Callable, List, Optional + +import platformdirs +from dagger import Client, Container +from pydantic import BaseSettings, Field, SecretBytes, SecretStr +from pygit2 import Commit, Repository #type: ignore + +from .singleton import Singleton + + +def get_git_revision() -> str: + repo = Repository(".") + commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) + return commit_hash + +def get_current_branch() -> str: + repo = Repository(".") + return str(repo.head.shorthand) + +def get_latest_commit_message() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.message) + +def get_latest_commit_author() -> str: + repo: Repository = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.author.name) + +def get_latest_commit_time() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.commit_time) + +def get_repo_root_path() -> str: + repo = Repository(".") + return str(os.path.dirname(os.path.dirname(repo.path))) + +def get_repo_fullname() -> str: + repo = Repository(".") + repo_url:str = repo.remotes["origin"].url + + # Handle HTTPS URLs + if "https://" in repo_url: + parts = repo_url.split("/") + owner = parts[-2] + repo_name = parts[-1].replace(".git", "") + + # Handle SSH URLs + else: + repo_url = repo_url.replace("git@github.com:", "") + owner, repo_name = repo_url.split("/")[:2] + repo_name = repo_name.replace(".git", "") + + return f"{owner}/{repo_name}" + +# Immutable. Use this for application configuration. Created at bootstrap. +class GlobalSettings(BaseSettings, Singleton): + DAGGER: bool = Field(True, env="DAGGER") + GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") + GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) + GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) + GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) + GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) + GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) + GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) + GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) + CI: bool = Field(False, env="CI") + LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") + PLATFORM: str = platform.system() + DEBUG: bool = Field(False, env="AIRCMD_DEBUG") + + # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") + GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") + GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") + GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") + GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") + GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") + GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") + GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") + GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") + GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") + + POETRY_CACHE_DIR: str = Field( + default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), + env="POETRY_CACHE_DIR" + ) + MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") + DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") + DEFAULT_EXCLUDED_FILES: List[str] = Field( + [ + ".git", + "**/build", + "**/.venv", + "**/secrets", + "**/__pycache__", + "**/*.egg-info", + "**/.vscode", + "**/.pytest_cache", + "**/.eggs", + "**/.mypy_cache", + "**/.DS_Store", + ], + env="DEFAULT_EXCLUDED_FILES" + ) + DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") + DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") + DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") + GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") + GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") + + PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") + PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") + PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") + + SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") + SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") + SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") + + PIP_CACHE_DIR: str = Field( + default_factory=lambda: platformdirs.user_cache_dir("pip"), + env="PIP_CACHE_DIR" + ) + + class Config: + arbitrary_types_allowed = True + env_file = '.env' + allow_mutation = False + + +''' +If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will +further filter the resulting environment variables based on the exclude list. + +Here's the order of operations: + + 1 If include is provided, only the environment variables with keys in the include list will be considered. + 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. + 3 The remaining environment variables will be loaded into the container. +''' + +def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: + def load_envs(ctr: Container) -> Container: + settings_dict = {key: value for key, value in settings.dict().items() if value is not None} + + if include is not None: + settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} + + if exclude is not None: + settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} + + for key, value in settings_dict.items(): + env_key = key.upper() + if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead + secret = client.set_secret(env_key, str(value.get_secret_value())) + ctr = ctr.with_secret_variable(env_key, secret) + else: + ctr = ctr.with_env_variable(env_key, str(value)) + + return ctr + + return load_envs + + +class GithubActionsInputSettings(BaseSettings): + """ + A Pydantic BaseSettings subclass that transforms input names to the format expected by GitHub Actions. + + GitHub Actions converts input names to environment variables in a specific way: + - The input name is converted to uppercase. + - Any '-' characters are converted to '_'. + - The prefix 'INPUT_' is added to the start. + + This class automatically applies these transformations when you create an instance of it. + + Example: + If you create an instance with the input {'project-token': 'abc'}, it will be transformed to {'INPUT_PROJECT_TOKEN': 'abc'}. + """ + + # Github action specific fields + GITHUB_ACTION: str + GITHUB_ACTOR: str + GITHUB_API_URL: str + GITHUB_EVENT_NAME: str + GITHUB_GRAPHQL_URL: str + GITHUB_JOB: str + GITHUB_REF: str + GITHUB_REPOSITORY: str + GITHUB_RUN_ID: str + GITHUB_RUN_NUMBER: str + GITHUB_SERVER_URL: str + GITHUB_SHA: str + GITHUB_EVENT_PATH: str + + class Config: + env_prefix = "INPUT_" + extra = "allow" + + def __init__(self, global_settings: GlobalSettings, **data: Any): + + # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. + + transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} + + # inject the context that github actions wants via environment variables. + # in typescript, it is injected here: + # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + + transformed_data.update({ + "GITHUB_SHA": global_settings.GIT_CURRENT_REVISION, + "GITHUB_REF": global_settings.GIT_CURRENT_BRANCH, + "GITHUB_EVENT_NAME": global_settings.GITHUB_EVENT_NAME, + "GITHUB_ACTION": global_settings.GITHUB_ACTION, + "GITHUB_ACTOR": global_settings.GITHUB_ACTOR, + "GITHUB_JOB": global_settings.GITHUB_JOB, + "GITHUB_RUN_NUMBER": global_settings.GITHUB_RUN_NUMBER, + "GITHUB_RUN_ID": global_settings.GITHUB_RUN_ID, + "GITHUB_API_URL": global_settings.GITHUB_API_URL, + "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, + "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, + "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, + "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH + }) + super().__init__(**transformed_data) + diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py new file mode 100644 index 000000000000..3fb7f62aedbd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py @@ -0,0 +1,22 @@ +from typing import Any, Type + + +class Singleton: + """ + A base class for implementing the Singleton pattern. + + This class stores instances and initialization flags for each subclass in dictionaries. + This allows each subclass to have its own unique instance and control over its initialization process. + + The __new__ method ensures that only one instance of each subclass is created. + The _initialized dictionary is used to control when the initialization logic of each subclass is executed. + """ + _instances: dict[Type['Singleton'], Any] = {} + _initialized: dict[Type['Singleton'], bool] = {} + + def __new__(cls: Type['Singleton'], *args: Any, **kwargs: Any) -> Any: + + if cls not in cls._instances: + cls._instances[cls] = super().__new__(cls) + cls._initialized[cls] = False + return cls._instances[cls] From efa0786d1214008a6e2cf636a3b6bff236c84ce8 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 19:01:57 -0700 Subject: [PATCH 004/141] Show demo of context coming through --- .../airbyte_ci/playground/commands.py | 24 +- .../pipelines/pipelines/stolen/base.py | 1 - .../pipelines/pipelines/stolen/settings.py | 238 +++++++++--------- airbyte-ci/connectors/pipelines/poetry.lock | 35 ++- .../connectors/pipelines/pyproject.toml | 1 + 5 files changed, 172 insertions(+), 127 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index ccff9988add5..9de4337cbec7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -16,14 +16,14 @@ # NEW @click.command() -@click.argument("arg1") +@click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context @pass_global_settings def playground( - ctx, - arg1: str, - opt: str, + ctx: PipelineContext, + args, + **kwargs, ): """Runs the tests for the given airbyte-ci package. @@ -31,5 +31,17 @@ def playground( poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ - print(f"playground: {arg1} {opt}") - print(f"ctx: {dir(ctx)}") + + # ctx = PipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) + # args = GlobalSettings(PLATFORM='Darwin') + # kwargs = {'opt': 'tight', 'hold': 'holdme'} + + import pdb; pdb.set_trace() + print(f"playground: {args} {kwargs}") + print(f"ctx: {ctx._click_context().obj}") + + # (Pdb) ctx._click_context().args + # [] + # (Pdb) ctx._click_context().params + # {'opt': 'tight', 'hold': 'holdme'} + # (Pdb) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index 40758a096587..ab3ef205feb3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -43,7 +43,6 @@ def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): """ if not Singleton._initialized[PipelineContext]: super().__init__(global_settings=global_settings, **data) - self.set_global_prefect_tag_context() Singleton._initialized[PipelineContext] = True import asyncio diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py index 16a20542c171..4596b6ba5d5c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py @@ -11,147 +11,147 @@ def get_git_revision() -> str: - repo = Repository(".") + repo = Repository(".") commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) return commit_hash -def get_current_branch() -> str: - repo = Repository(".") - return str(repo.head.shorthand) - -def get_latest_commit_message() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.message) - -def get_latest_commit_author() -> str: - repo: Repository = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.author.name) - -def get_latest_commit_time() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.commit_time) +def get_current_branch() -> str: + repo = Repository(".") + return str(repo.head.shorthand) + +def get_latest_commit_message() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.message) + +def get_latest_commit_author() -> str: + repo: Repository = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.author.name) + +def get_latest_commit_time() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.commit_time) def get_repo_root_path() -> str: repo = Repository(".") return str(os.path.dirname(os.path.dirname(repo.path))) -def get_repo_fullname() -> str: +def get_repo_fullname() -> str: repo = Repository(".") repo_url:str = repo.remotes["origin"].url - + # Handle HTTPS URLs if "https://" in repo_url: parts = repo_url.split("/") owner = parts[-2] repo_name = parts[-1].replace(".git", "") - + # Handle SSH URLs else: repo_url = repo_url.replace("git@github.com:", "") owner, repo_name = repo_url.split("/")[:2] repo_name = repo_name.replace(".git", "") - + return f"{owner}/{repo_name}" # Immutable. Use this for application configuration. Created at bootstrap. class GlobalSettings(BaseSettings, Singleton): - DAGGER: bool = Field(True, env="DAGGER") - GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") - GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) - GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) - GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) - GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) - GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) - GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) - GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) - CI: bool = Field(False, env="CI") - LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") + # DAGGER: bool = Field(True, env="DAGGER") + # GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") + # GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) + # GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) + # GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) + # GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) + # GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) + # GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) + # GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) + # CI: bool = Field(False, env="CI") + # LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") PLATFORM: str = platform.system() - DEBUG: bool = Field(False, env="AIRCMD_DEBUG") - - # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") - GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") - GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") - GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") - GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") - GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") - GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") - GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") - GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") - GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") - - POETRY_CACHE_DIR: str = Field( - default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), - env="POETRY_CACHE_DIR" - ) - MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") - DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") - DEFAULT_EXCLUDED_FILES: List[str] = Field( - [ - ".git", - "**/build", - "**/.venv", - "**/secrets", - "**/__pycache__", - "**/*.egg-info", - "**/.vscode", - "**/.pytest_cache", - "**/.eggs", - "**/.mypy_cache", - "**/.DS_Store", - ], - env="DEFAULT_EXCLUDED_FILES" - ) - DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") - DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") - DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") - GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") - GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") - - PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") - PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") - PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") - - SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") - SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") - SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") - - PIP_CACHE_DIR: str = Field( - default_factory=lambda: platformdirs.user_cache_dir("pip"), - env="PIP_CACHE_DIR" - ) - - class Config: - arbitrary_types_allowed = True - env_file = '.env' + # DEBUG: bool = Field(False, env="AIRCMD_DEBUG") + + # # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + # GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") + # GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") + # GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") + # GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") + # GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") + # GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") + # GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") + # GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") + # GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") + # GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") + + # POETRY_CACHE_DIR: str = Field( + # default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), + # env="POETRY_CACHE_DIR" + # ) + # MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") + # DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") + # DEFAULT_EXCLUDED_FILES: List[str] = Field( + # [ + # ".git", + # "**/build", + # "**/.venv", + # "**/secrets", + # "**/__pycache__", + # "**/*.egg-info", + # "**/.vscode", + # "**/.pytest_cache", + # "**/.eggs", + # "**/.mypy_cache", + # "**/.DS_Store", + # ], + # env="DEFAULT_EXCLUDED_FILES" + # ) + # DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") + # DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") + # DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") + # GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") + # GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") + + # PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") + # PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") + # PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") + + # SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") + # SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") + # SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") + + # PIP_CACHE_DIR: str = Field( + # default_factory=lambda: platformdirs.user_cache_dir("pip"), + # env="PIP_CACHE_DIR" + # ) + + class Config: + arbitrary_types_allowed = True + # env_file = '.env' allow_mutation = False ''' -If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will -further filter the resulting environment variables based on the exclude list. - -Here's the order of operations: - - 1 If include is provided, only the environment variables with keys in the include list will be considered. - 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. - 3 The remaining environment variables will be loaded into the container. -''' - -def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: - def load_envs(ctr: Container) -> Container: - settings_dict = {key: value for key, value in settings.dict().items() if value is not None} - - if include is not None: - settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} - - if exclude is not None: - settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} - +If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will +further filter the resulting environment variables based on the exclude list. + +Here's the order of operations: + + 1 If include is provided, only the environment variables with keys in the include list will be considered. + 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. + 3 The remaining environment variables will be loaded into the container. +''' + +def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: + def load_envs(ctr: Container) -> Container: + settings_dict = {key: value for key, value in settings.dict().items() if value is not None} + + if include is not None: + settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} + + if exclude is not None: + settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} + for key, value in settings_dict.items(): env_key = key.upper() if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead @@ -159,10 +159,10 @@ def load_envs(ctr: Container) -> Container: ctr = ctr.with_secret_variable(env_key, secret) else: ctr = ctr.with_env_variable(env_key, str(value)) - - return ctr - - return load_envs + + return ctr + + return load_envs class GithubActionsInputSettings(BaseSettings): @@ -197,14 +197,14 @@ class GithubActionsInputSettings(BaseSettings): class Config: env_prefix = "INPUT_" - extra = "allow" + extra = "allow" def __init__(self, global_settings: GlobalSettings, **data: Any): # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} - + # inject the context that github actions wants via environment variables. # in typescript, it is injected here: # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 @@ -221,7 +221,7 @@ def __init__(self, global_settings: GlobalSettings, **data: Any): "GITHUB_API_URL": global_settings.GITHUB_API_URL, "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, - "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, + "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH }) super().__init__(**transformed_data) diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 68b610e709d2..4dfc6816e6d5 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1525,6 +1525,39 @@ typing-extensions = ">=3.10,<4.6.0 || >4.6.0" [package.extras] dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] +[[package]] +name = "pygit2" +version = "1.13.1" +description = "Python bindings for libgit2." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygit2-1.13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:30db67f73ef28b07864f2509978b7396d1ed3b72f6f252d301db12a4c9f90f5b"}, + {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825f22a1bbf73c7a11c69e53a29485d10b4df6a635ccd120cf2966e6535a5b52"}, + {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7356d4e41f122a066fa1cce3f5dbedf73c03781692f5eab3687bc355a083575"}, + {file = "pygit2-1.13.1-cp310-cp310-win32.whl", hash = "sha256:cf47de2e21cdeb5d8c35f0d1a381b56fdb365dac3dcd8ea7fa057b390ce83d40"}, + {file = "pygit2-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:6ede9700fdbf78a5a1513549f37884233f29d3343412272c0800cda40c4c2c56"}, + {file = "pygit2-1.13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9c84eff2223e5fd442b746785b9cd21f98c1f53a0f3fe8d4ed06aee60a09ea35"}, + {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd649b9ef17564b642f59e1a2751e30fdd07d3707b0642d8012062615651039"}, + {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:787ea717bb9fadb3ee2836ed32a9ed2110ef861862bfe6b693becda75a2eaa5c"}, + {file = "pygit2-1.13.1-cp311-cp311-win32.whl", hash = "sha256:d2dbf3d6976b0626fafd7d1c7363ae92dcacaa63789e8c432bc8caea86132235"}, + {file = "pygit2-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce8618e5876b4c54942587d72a0d84f6e6a5b0e69db5f8d06dc5f567abd07ed1"}, + {file = "pygit2-1.13.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8d2d97cfe2bf2abbb0ef5984771578d1b05053942bfe1b46d4ac48d19c5eda56"}, + {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a162db1afdc5bae608d739395a248a373165176f83c7fe57a1073e9168b459"}, + {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357c11b30d6c63ff58401a897df39b27903beffebb24842c8ce9ce77b90fe0b1"}, + {file = "pygit2-1.13.1-cp38-cp38-win32.whl", hash = "sha256:ed7fc70bc8f6db227c9919958d064cb49eaa68cc97f51c1f9de920a4500c6766"}, + {file = "pygit2-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:949ad31e0fab408449721cc5b582350f6c5c56ab068bfa10cd6d10c2830deaa9"}, + {file = "pygit2-1.13.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ea2b870675ef1a2bef3300dda725aae9f8c68265e633ed683fce85588cfb4d37"}, + {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74af678b98f6a08ef4315f5b64889011e05ad702e340cc6cde59926906650039"}, + {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33bbe4d7501a600be320147c63a8d1a966fb7424595e6eb53fdc30259b8921dc"}, + {file = "pygit2-1.13.1-cp39-cp39-win32.whl", hash = "sha256:697044df77c8b3849fec8d7dd454acd347b180212c6bc5526eeb9309eff63a65"}, + {file = "pygit2-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:8004244da8183fcefcf7c3d4d119806e9c705543bcf24045b97e3eddaa869aef"}, + {file = "pygit2-1.13.1.tar.gz", hash = "sha256:d8e6d540aad9ded1cf2c6bda31ba48b1e20c18525807dbd837317bef4dccb994"}, +] + +[package.dependencies] +cffi = ">=1.9.1" + [[package]] name = "pygithub" version = "1.59.1" @@ -2248,4 +2281,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "76b53958ee6dd19c2816b023cce1585bb198e14c79124d56de2445c494d51cf2" +content-hash = "7b6c32239940ee2c73e44c3f75267a56f46d7a513a14efe676b7a6da4c74f8c3" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 84d8dabca86d..bb70ce815cc2 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -25,6 +25,7 @@ connector-ops = {path = "../connector_ops", develop = true} toml = "^0.10.2" sentry-sdk = "^1.28.1" segment-analytics-python = "^2.2.3" +pygit2 = "^1.13.1" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 76458dc154f954cb54d14e961e9ab986a6dffe2a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 13:11:18 -0700 Subject: [PATCH 005/141] Add pass_click_context_and_args_to_children --- .../airbyte_ci/playground/commands.py | 14 +++--- .../pipelines/pipelines/cli/airbyte_ci.py | 49 ++++++++++++++----- .../pipelines/pipelines/stolen/base.py | 30 +++++++++--- 3 files changed, 65 insertions(+), 28 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 9de4337cbec7..5c3580617a54 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,15 +3,15 @@ # import click -from pipelines.stolen.base import PipelineContext +from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator from pipelines.stolen.settings import GlobalSettings # Stolen settings = GlobalSettings() -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(PipelineContext, global_settings=settings) -pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext, global_settings=settings) +# pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) # NEW @@ -19,10 +19,9 @@ @click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context -@pass_global_settings +# @pass_global_settings def playground( - ctx: PipelineContext, - args, + ctx: ClickPipelineContext, **kwargs, ): """Runs the tests for the given airbyte-ci package. @@ -32,12 +31,11 @@ def playground( test_directory (str): The directory containing the tests to run. """ - # ctx = PipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) + # ctx = ClickPipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) # args = GlobalSettings(PLATFORM='Darwin') # kwargs = {'opt': 'tight', 'hold': 'holdme'} import pdb; pdb.set_trace() - print(f"playground: {args} {kwargs}") print(f"ctx: {ctx._click_context().obj}") # (Pdb) ctx._click_context().args diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index e1b55fefa8ab..8d96e9ecc5ca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -146,6 +146,28 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) +# TO MOVE + +def pass_click_context_and_args_to_children(f): + """ + TODO + """ + + def wrapper(*args, **kwargs): + ctx = args[0] + ctx.ensure_object(dict) + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + return f(*args, **kwargs) + + return wrapper # COMMANDS @@ -187,6 +209,7 @@ def get_modified_files( @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) @click.pass_context +@pass_click_context_and_args_to_children @track_command def airbyte_ci( ctx: click.Context, @@ -206,24 +229,24 @@ def airbyte_ci( show_dagger_logs: bool, ): # noqa D103 display_welcome_message() - ctx.ensure_object(dict) + check_up_to_date() - ctx.obj["is_local"] = is_local + # ctx.obj["is_local"] = is_local ctx.obj["is_ci"] = not is_local - ctx.obj["git_branch"] = git_branch - ctx.obj["git_revision"] = git_revision - ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id + # ctx.obj["git_branch"] = git_branch + # ctx.obj["git_revision"] = git_revision + # ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id ctx.obj["gha_workflow_run_url"] = ( f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None ) - ctx.obj["ci_context"] = ci_context - ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name - ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials - ctx.obj["ci_git_user"] = ci_git_user - ctx.obj["ci_github_access_token"] = ci_github_access_token - ctx.obj["ci_job_key"] = ci_job_key - ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp - ctx.obj["show_dagger_logs"] = show_dagger_logs + # ctx.obj["ci_context"] = ci_context + # ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name + # ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials + # ctx.obj["ci_git_user"] = ci_git_user + # ctx.obj["ci_github_access_token"] = ci_github_access_token + # ctx.obj["ci_job_key"] = ci_job_key + # ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp + # ctx.obj["show_dagger_logs"] = show_dagger_logs if pull_request_number and ci_github_access_token: ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index ab3ef205feb3..188e9639d72d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -21,29 +21,45 @@ def get_context() -> Context: return get_current_context() -class PipelineContext(BaseModel, Singleton): +class ClickPipelineContext(BaseModel, Singleton): global_settings: GlobalSettings dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + @property + def params(self): + ctx = self._click_context() + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + + # Error if click_obj and click_params have the same key + all_click_params_keys = [p.name for p in ctx.command.params.keys] + intersection = set(click_obj.keys()) & set(all_click_params_keys) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + return {**click_obj, **click_params} + class Config: arbitrary_types_allowed=True def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): """ - Initialize the PipelineContext instance. + Initialize the ClickPipelineContext instance. - This method checks the _initialized flag for the PipelineContext class in the Singleton base class. + This method checks the _initialized flag for the ClickPipelineContext class in the Singleton base class. If the flag is False, the initialization logic is executed and the flag is set to True. If the flag is True, the initialization logic is skipped. - This ensures that the initialization logic is only executed once, even if the PipelineContext instance is retrieved multiple times. + This ensures that the initialization logic is only executed once, even if the ClickPipelineContext instance is retrieved multiple times. This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). """ - if not Singleton._initialized[PipelineContext]: + if not Singleton._initialized[ClickPipelineContext]: super().__init__(global_settings=global_settings, **data) - Singleton._initialized[PipelineContext] = True + Singleton._initialized[ClickPipelineContext] = True import asyncio @@ -61,7 +77,7 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name class GlobalContext(BaseModel, Singleton): - pipeline_context: Optional[PipelineContext] = Field(default=None) + pipeline_context: Optional[ClickPipelineContext] = Field(default=None) click_context: Optional[Context] = Field(default=None) class Config: From 58845d36175a90bea9e0c8d3d44c076d83e7f88a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 13:56:37 -0700 Subject: [PATCH 006/141] Add click decorator to reduce boilerplate --- .../connectors/build_image/commands.py | 2 +- .../connectors/bump_version/commands.py | 2 +- .../airbyte_ci/connectors/commands.py | 13 +- .../airbyte_ci/connectors/context.py | 8 +- .../migrate_to_base_image/commands.py | 2 +- .../airbyte_ci/connectors/reports.py | 4 +- .../connectors/upgrade_base_image/commands.py | 2 +- .../airbyte_ci/playground/commands.py | 12 +- .../pipelines/pipelines/cli/airbyte_ci.py | 82 ++----- .../pipelines/cli/click_decorators.py | 68 ++++++ .../pipelines/pipelines/cli/telemetry.py | 6 +- .../pipelines/pipelines/models/contexts.py | 4 +- .../pipelines/pipelines/stolen/base.py | 16 +- .../pipelines/pipelines/stolen/settings.py | 228 ------------------ 14 files changed, 113 insertions(+), 336 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py index 0cdf469c9d9c..d391b2f62a25 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -38,7 +38,7 @@ def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool: ci_context=ctx.obj.get("ci_context"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], use_local_cdk=ctx.obj.get("use_local_cdk"), - open_report_in_browser=ctx.obj.get("open_report_in_browser"), + enable_report_auto_open=ctx.obj.get("enable_report_auto_open"), use_host_gradle_dist_tar=use_host_gradle_dist_tar, ) for connector in ctx.obj["selected_connectors_with_modified_files"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py index 1da52905c82d..d93b9e4385a7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py @@ -40,7 +40,7 @@ def bump_version( ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index f985348db32a..58796ee414e7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,6 +9,7 @@ import click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -177,7 +178,8 @@ def validate_environment(is_local: bool, use_remote_secrets: bool): default=True, type=bool, ) -@click.pass_context +@click_pass_context_and_args_to_children +@click_ignore_unused_kwargs def connectors( ctx: click.Context, use_remote_secrets: bool, @@ -187,21 +189,12 @@ def connectors( modified: bool, metadata_changes_only: bool, metadata_query: str, - concurrency: int, - execute_timeout: int, enable_dependency_scanning: bool, - use_local_cdk: bool, - enable_report_auto_open: bool, ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"], use_remote_secrets) ctx.ensure_object(dict) - ctx.obj["use_remote_secrets"] = use_remote_secrets - ctx.obj["concurrency"] = concurrency - ctx.obj["execute_timeout"] = execute_timeout - ctx.obj["use_local_cdk"] = use_local_cdk - ctx.obj["open_report_in_browser"] = enable_report_auto_open ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( names, support_levels, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 1c7dc0615509..db39c29b9655 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -54,7 +54,7 @@ def __init__( code_tests_only: bool = False, use_local_cdk: bool = False, use_host_gradle_dist_tar: bool = False, - open_report_in_browser: bool = True, + enable_report_auto_open: bool = True, docker_hub_username: Optional[str] = None, docker_hub_password: Optional[str] = None, ): @@ -79,7 +79,7 @@ def __init__( fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False. - open_report_in_browser (bool, optional): Open HTML report in browser window. Defaults to True. + enable_report_auto_open (bool, optional): Open HTML report in browser window. Defaults to True. docker_hub_username (Optional[str], optional): Docker Hub username to use to read registries. Defaults to None. docker_hub_password (Optional[str], optional): Docker Hub password to use to read registries. Defaults to None. """ @@ -98,7 +98,7 @@ def __init__( self.code_tests_only = code_tests_only self.use_local_cdk = use_local_cdk self.use_host_gradle_dist_tar = use_host_gradle_dist_tar - self.open_report_in_browser = open_report_in_browser + self.enable_report_auto_open = enable_report_auto_open self.docker_hub_username = docker_hub_username self.docker_hub_password = docker_hub_password @@ -118,7 +118,7 @@ def __init__( ci_gcs_credentials=ci_gcs_credentials, ci_git_user=ci_git_user, ci_github_access_token=ci_github_access_token, - open_report_in_browser=open_report_in_browser, + enable_report_auto_open=enable_report_auto_open, ) @property diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py index b57afc0e0005..1ce65625a971 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py @@ -55,7 +55,7 @@ def migrate_to_base_image( ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 4762f508d30d..2b64f13eaf91 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -139,9 +139,9 @@ async def to_html(self) -> str: async def save(self) -> None: local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) absolute_path = await local_html_path.resolve() - if self.pipeline_context.open_report_in_browser: + if self.pipeline_context.enable_report_auto_open: self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") - if self.pipeline_context.open_report_in_browser: + if self.pipeline_context.enable_report_auto_open: self.pipeline_context.logger.info("Opening HTML report in browser.") webbrowser.open(absolute_path.as_uri()) if self.remote_storage_enabled: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py index 7c857bf617a4..e7f4e36bd883 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py @@ -47,7 +47,7 @@ def upgrade_base_image(ctx: click.Context, set_if_not_exists: bool, docker_hub_u ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 5c3580617a54..77640f7c426d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,26 +3,20 @@ # import click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator -from pipelines.stolen.settings import GlobalSettings -# Stolen -settings = GlobalSettings() -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext, global_settings=settings) -# pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) - -# NEW +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() @click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context -# @pass_global_settings +@click_ignore_unused_kwargs def playground( ctx: ClickPipelineContext, - **kwargs, ): """Runs the tests for the given airbyte-ci package. diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 8d96e9ecc5ca..9a2e5bb00eec 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -14,8 +14,9 @@ import git from github import PullRequest from pipelines import main_logger +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.cli.lazy_group import LazyGroup -from pipelines.cli.telemetry import track_command +from pipelines.cli.telemetry import click_track_command from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext from pipelines.helpers import github from pipelines.helpers.git import ( @@ -146,32 +147,20 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) -# TO MOVE - -def pass_click_context_and_args_to_children(f): - """ - TODO - """ - - def wrapper(*args, **kwargs): - ctx = args[0] - ctx.ensure_object(dict) - click_obj = ctx.obj - click_params = ctx.params - command_name = ctx.command.name - - # Error if click_obj and click_params have the same key - intersection = set(click_obj.keys()) & set(click_params.keys()) - if intersection: - raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") - - return f(*args, **kwargs) - - return wrapper +def log_git_info(ctx: click.Context): + main_logger.info("Running airbyte-ci in CI mode.") + main_logger.info(f"CI Context: {ctx.obj['ci_context']}") + main_logger.info(f"CI Report Bucket Name: {ctx.obj['ci_report_bucket_name']}") + main_logger.info(f"Git Branch: {ctx.obj['git_branch']}") + main_logger.info(f"Git Revision: {ctx.obj['git_revision']}") + main_logger.info(f"GitHub Workflow Run ID: {ctx.obj['gha_workflow_run_id']}") + main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") + main_logger.info(f"Pull Request Number: {ctx.obj['pull_request_number']}") + main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") + main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") # COMMANDS - @click.group( cls=LazyGroup, help="Airbyte CI top-level command group.", @@ -208,9 +197,9 @@ def wrapper(*args, **kwargs): ) @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) -@click.pass_context -@pass_click_context_and_args_to_children -@track_command +@click_track_command +@click_pass_context_and_args_to_children +@click_ignore_unused_kwargs def airbyte_ci( ctx: click.Context, is_local: bool, @@ -219,55 +208,26 @@ def airbyte_ci( diffed_branch: str, gha_workflow_run_id: str, ci_context: str, - pipeline_start_timestamp: int, pull_request_number: int, - ci_git_user: str, ci_github_access_token: str, - ci_report_bucket_name: str, - ci_gcs_credentials: str, - ci_job_key: str, - show_dagger_logs: bool, ): # noqa D103 display_welcome_message() - check_up_to_date() - # ctx.obj["is_local"] = is_local + ctx.obj["is_ci"] = not is_local - # ctx.obj["git_branch"] = git_branch - # ctx.obj["git_revision"] = git_revision - # ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id ctx.obj["gha_workflow_run_url"] = ( f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None ) - # ctx.obj["ci_context"] = ci_context - # ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name - # ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials - # ctx.obj["ci_git_user"] = ci_git_user - # ctx.obj["ci_github_access_token"] = ci_github_access_token - # ctx.obj["ci_job_key"] = ci_job_key - # ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp - # ctx.obj["show_dagger_logs"] = show_dagger_logs - - if pull_request_number and ci_github_access_token: - ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) - else: - ctx.obj["pull_request"] = None + + can_get_pull_request = pull_request_number and ci_github_access_token + ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) if can_get_pull_request else None ctx.obj["modified_files"] = transform_strs_to_paths( get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) ) if not is_local: - main_logger.info("Running airbyte-ci in CI mode.") - main_logger.info(f"CI Context: {ci_context}") - main_logger.info(f"CI Report Bucket Name: {ci_report_bucket_name}") - main_logger.info(f"Git Branch: {git_branch}") - main_logger.info(f"Git Revision: {git_revision}") - main_logger.info(f"GitHub Workflow Run ID: {gha_workflow_run_id}") - main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") - main_logger.info(f"Pull Request Number: {pull_request_number}") - main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}") - main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") + log_git_info(ctx) set_working_directory_to_root() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py new file mode 100644 index 000000000000..97b08e635107 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -0,0 +1,68 @@ + + +import functools +import inspect +import click + + + +def _contains_var_kwarg(f): + return any( + param.kind == inspect.Parameter.VAR_KEYWORD + for param in inspect.signature(f).parameters.values() + ) + + +def _is_kwarg_of(key, f): + param = inspect.signature(f).parameters.get(key, False) + return param and ( + param.kind is inspect.Parameter.KEYWORD_ONLY or + param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + ) + + +def click_ignore_unused_kwargs(f): + """Make function ignore unmatched kwargs. + + If the function already has the catch all **kwargs, do nothing. + + Useful in the case that the argument is meant to be passed to a child command + and is not used by the parent command + """ + if _contains_var_kwarg(f): + return f + + @functools.wraps(f) + def inner(*args, **kwargs): + filtered_kwargs = { + key: value + for key, value in kwargs.items() + if _is_kwarg_of(key, f) + } + return f(*args, **filtered_kwargs) + return inner + + +def click_pass_context_and_args_to_children(f): + """ + Decorator to pass click context and args to children commands. + """ + + @click.pass_context + def wrapper(*args, **kwargs): + ctx = args[0] + ctx.ensure_object(dict) + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + ctx.obj = {**click_obj, **click_params} + + return f(*args, **kwargs) + + return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 337a9182987d..942d69cb5751 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -9,6 +9,7 @@ import sys import segment.analytics as analytics +from click import get_current_context analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" analytics.send = True @@ -34,13 +35,12 @@ def _get_anonymous_system_id(): return unique_id -def track_command(f): +def click_track_command(f): """ Decorator to track CLI commands with segment.io """ - def wrapper(*args, **kwargs): - ctx = args[0] + ctx = get_current_context() top_level_command = ctx.command_path full_cmd = " ".join(sys.argv) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py index 69a47f540f1c..72a858145476 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py @@ -62,7 +62,7 @@ def __init__( ci_gcs_credentials: Optional[str] = None, ci_git_user: Optional[str] = None, ci_github_access_token: Optional[str] = None, - open_report_in_browser: bool = True, + enable_report_auto_open: bool = True, ): """Initialize a pipeline context. @@ -105,7 +105,7 @@ def __init__( self.started_at = None self.stopped_at = None self.secrets_to_mask = [] - self.open_report_in_browser = open_report_in_browser + self.enable_report_auto_open = enable_report_auto_open update_commit_status_check(**self.github_commit_status) @property diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index 188e9639d72d..efcf51ed7bca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -10,7 +10,6 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr -from .settings import GlobalSettings from .singleton import Singleton @@ -22,7 +21,6 @@ def get_context() -> Context: return get_current_context() class ClickPipelineContext(BaseModel, Singleton): - global_settings: GlobalSettings dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) @@ -34,9 +32,8 @@ def params(self): click_params = ctx.params command_name = ctx.command.name - # Error if click_obj and click_params have the same key - all_click_params_keys = [p.name for p in ctx.command.params.keys] + all_click_params_keys = [p.name for p in ctx.command.params] intersection = set(click_obj.keys()) & set(all_click_params_keys) if intersection: raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") @@ -46,7 +43,7 @@ def params(self): class Config: arbitrary_types_allowed=True - def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): + def __init__(self, **data: dict[str, Any]): """ Initialize the ClickPipelineContext instance. @@ -58,7 +55,7 @@ def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). """ if not Singleton._initialized[ClickPipelineContext]: - super().__init__(global_settings=global_settings, **data) + super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True import asyncio @@ -75,10 +72,3 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client - -class GlobalContext(BaseModel, Singleton): - pipeline_context: Optional[ClickPipelineContext] = Field(default=None) - click_context: Optional[Context] = Field(default=None) - - class Config: - arbitrary_types_allowed = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py deleted file mode 100644 index 4596b6ba5d5c..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py +++ /dev/null @@ -1,228 +0,0 @@ -import os -import platform -from typing import Any, Callable, List, Optional - -import platformdirs -from dagger import Client, Container -from pydantic import BaseSettings, Field, SecretBytes, SecretStr -from pygit2 import Commit, Repository #type: ignore - -from .singleton import Singleton - - -def get_git_revision() -> str: - repo = Repository(".") - commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) - return commit_hash - -def get_current_branch() -> str: - repo = Repository(".") - return str(repo.head.shorthand) - -def get_latest_commit_message() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.message) - -def get_latest_commit_author() -> str: - repo: Repository = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.author.name) - -def get_latest_commit_time() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.commit_time) - -def get_repo_root_path() -> str: - repo = Repository(".") - return str(os.path.dirname(os.path.dirname(repo.path))) - -def get_repo_fullname() -> str: - repo = Repository(".") - repo_url:str = repo.remotes["origin"].url - - # Handle HTTPS URLs - if "https://" in repo_url: - parts = repo_url.split("/") - owner = parts[-2] - repo_name = parts[-1].replace(".git", "") - - # Handle SSH URLs - else: - repo_url = repo_url.replace("git@github.com:", "") - owner, repo_name = repo_url.split("/")[:2] - repo_name = repo_name.replace(".git", "") - - return f"{owner}/{repo_name}" - -# Immutable. Use this for application configuration. Created at bootstrap. -class GlobalSettings(BaseSettings, Singleton): - # DAGGER: bool = Field(True, env="DAGGER") - # GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") - # GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) - # GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) - # GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) - # GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) - # GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) - # GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) - # GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) - # CI: bool = Field(False, env="CI") - # LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") - PLATFORM: str = platform.system() - # DEBUG: bool = Field(False, env="AIRCMD_DEBUG") - - # # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - # GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") - # GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") - # GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") - # GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") - # GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") - # GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") - # GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") - # GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") - # GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") - # GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") - - # POETRY_CACHE_DIR: str = Field( - # default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), - # env="POETRY_CACHE_DIR" - # ) - # MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") - # DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") - # DEFAULT_EXCLUDED_FILES: List[str] = Field( - # [ - # ".git", - # "**/build", - # "**/.venv", - # "**/secrets", - # "**/__pycache__", - # "**/*.egg-info", - # "**/.vscode", - # "**/.pytest_cache", - # "**/.eggs", - # "**/.mypy_cache", - # "**/.DS_Store", - # ], - # env="DEFAULT_EXCLUDED_FILES" - # ) - # DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") - # DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") - # DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") - # GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") - # GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") - - # PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") - # PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") - # PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") - - # SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") - # SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") - # SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") - - # PIP_CACHE_DIR: str = Field( - # default_factory=lambda: platformdirs.user_cache_dir("pip"), - # env="PIP_CACHE_DIR" - # ) - - class Config: - arbitrary_types_allowed = True - # env_file = '.env' - allow_mutation = False - - -''' -If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will -further filter the resulting environment variables based on the exclude list. - -Here's the order of operations: - - 1 If include is provided, only the environment variables with keys in the include list will be considered. - 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. - 3 The remaining environment variables will be loaded into the container. -''' - -def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: - def load_envs(ctr: Container) -> Container: - settings_dict = {key: value for key, value in settings.dict().items() if value is not None} - - if include is not None: - settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} - - if exclude is not None: - settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} - - for key, value in settings_dict.items(): - env_key = key.upper() - if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead - secret = client.set_secret(env_key, str(value.get_secret_value())) - ctr = ctr.with_secret_variable(env_key, secret) - else: - ctr = ctr.with_env_variable(env_key, str(value)) - - return ctr - - return load_envs - - -class GithubActionsInputSettings(BaseSettings): - """ - A Pydantic BaseSettings subclass that transforms input names to the format expected by GitHub Actions. - - GitHub Actions converts input names to environment variables in a specific way: - - The input name is converted to uppercase. - - Any '-' characters are converted to '_'. - - The prefix 'INPUT_' is added to the start. - - This class automatically applies these transformations when you create an instance of it. - - Example: - If you create an instance with the input {'project-token': 'abc'}, it will be transformed to {'INPUT_PROJECT_TOKEN': 'abc'}. - """ - - # Github action specific fields - GITHUB_ACTION: str - GITHUB_ACTOR: str - GITHUB_API_URL: str - GITHUB_EVENT_NAME: str - GITHUB_GRAPHQL_URL: str - GITHUB_JOB: str - GITHUB_REF: str - GITHUB_REPOSITORY: str - GITHUB_RUN_ID: str - GITHUB_RUN_NUMBER: str - GITHUB_SERVER_URL: str - GITHUB_SHA: str - GITHUB_EVENT_PATH: str - - class Config: - env_prefix = "INPUT_" - extra = "allow" - - def __init__(self, global_settings: GlobalSettings, **data: Any): - - # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. - - transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} - - # inject the context that github actions wants via environment variables. - # in typescript, it is injected here: - # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - - transformed_data.update({ - "GITHUB_SHA": global_settings.GIT_CURRENT_REVISION, - "GITHUB_REF": global_settings.GIT_CURRENT_BRANCH, - "GITHUB_EVENT_NAME": global_settings.GITHUB_EVENT_NAME, - "GITHUB_ACTION": global_settings.GITHUB_ACTION, - "GITHUB_ACTOR": global_settings.GITHUB_ACTOR, - "GITHUB_JOB": global_settings.GITHUB_JOB, - "GITHUB_RUN_NUMBER": global_settings.GITHUB_RUN_NUMBER, - "GITHUB_RUN_ID": global_settings.GITHUB_RUN_ID, - "GITHUB_API_URL": global_settings.GITHUB_API_URL, - "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, - "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, - "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, - "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH - }) - super().__init__(**transformed_data) - From 435f44dd29c5e5ffa815ac9f8c906bb821d47a15 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 14:22:43 -0700 Subject: [PATCH 007/141] Add click_append_to_context_object decorator --- .../airbyte_ci/connectors/commands.py | 5 +- .../airbyte_ci/playground/commands.py | 5 +- .../pipelines/pipelines/cli/airbyte_ci.py | 56 +++++++++++-------- .../pipelines/cli/click_decorators.py | 21 ++++++- 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 58796ee414e7..dcb14b318e1d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -178,7 +178,7 @@ def validate_environment(is_local: bool, use_remote_secrets: bool): default=True, type=bool, ) -@click_pass_context_and_args_to_children +@click_merge_args_into_context_obj @click_ignore_unused_kwargs def connectors( ctx: click.Context, @@ -194,7 +194,6 @@ def connectors( """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"], use_remote_secrets) - ctx.ensure_object(dict) ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( names, support_levels, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 77640f7c426d..4cd43340e07a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,7 +3,7 @@ # import click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator @@ -29,8 +29,7 @@ def playground( # args = GlobalSettings(PLATFORM='Darwin') # kwargs = {'opt': 'tight', 'hold': 'holdme'} - import pdb; pdb.set_trace() - print(f"ctx: {ctx._click_context().obj}") + print(f"params: {ctx.params}") # (Pdb) ctx._click_context().args # [] diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 9a2e5bb00eec..2be63e30698b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -8,13 +8,13 @@ import logging import os from pathlib import Path -from typing import List +from typing import List, Optional import click import git from github import PullRequest from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.cli.telemetry import click_track_command from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext @@ -159,6 +159,33 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") +def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: + if ctx.obj["is_local"]: + return "TESTEST" + return ctx.obj["gha_workflow_run_id"] + +def _get_pull_request(ctx: click.Context): + pull_request_number = ctx.obj["pull_request_number"] + ci_github_access_token = ctx.obj["ci_github_access_token"] + + can_get_pull_request = pull_request_number and ci_github_access_token + if not can_get_pull_request: + return None + + return github.get_pull_request(pull_request_number, ci_github_access_token) + +def _get_modified_files(ctx: click.Context) -> List[Path]: + return transform_strs_to_paths( + get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"] + ) + ) + # COMMANDS @click.group( @@ -198,34 +225,19 @@ def log_git_info(ctx: click.Context): @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) @click_track_command -@click_pass_context_and_args_to_children +@click_merge_args_into_context_obj +@click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) +@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) +@click_append_to_context_object("pull_request", _get_pull_request) +@click_append_to_context_object("modified_files", _get_modified_files) @click_ignore_unused_kwargs def airbyte_ci( ctx: click.Context, is_local: bool, - git_branch: str, - git_revision: str, - diffed_branch: str, - gha_workflow_run_id: str, - ci_context: str, - pull_request_number: int, - ci_github_access_token: str, ): # noqa D103 display_welcome_message() check_up_to_date() - ctx.obj["is_ci"] = not is_local - ctx.obj["gha_workflow_run_url"] = ( - f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None - ) - - can_get_pull_request = pull_request_number and ci_github_access_token - ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) if can_get_pull_request else None - - ctx.obj["modified_files"] = transform_strs_to_paths( - get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) - ) - if not is_local: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 97b08e635107..dcd6516c14da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -2,6 +2,7 @@ import functools import inspect +from typing import Any, Callable import click @@ -43,7 +44,7 @@ def inner(*args, **kwargs): return inner -def click_pass_context_and_args_to_children(f): +def click_merge_args_into_context_obj(f): """ Decorator to pass click context and args to children commands. """ @@ -66,3 +67,21 @@ def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper + +def click_append_to_context_object(key: str, value: Callable | Any): + """ + Decorator to append a value to the click context object. + """ + + def decorator(f): + def wrapper(*args, **kwargs): + ctx = click.get_current_context() + ctx.ensure_object(dict) + + if callable(value): + ctx.obj[key] = value(ctx) + else: + ctx.obj[key] = value + return f(*args, **kwargs) + return wrapper + return decorator From 55baf8f6c1660f5f512e9f7024827f82877701f3 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 14:35:32 -0700 Subject: [PATCH 008/141] Move files --- .../airbyte_ci/connectors/context.py | 2 +- .../airbyte_ci/playground/commands.py | 59 ++++++++++++++----- .../pipelines/airbyte_ci/steps/docker.py | 2 +- .../pipelines/airbyte_ci/steps/gradle.py | 2 +- .../pipelines/airbyte_ci/steps/no_op.py | 2 +- .../pipelines/airbyte_ci/steps/poetry.py | 2 +- .../pipelines/cli/click_decorators.py | 26 +++++++- .../pipelines/models/contexts/__init__.py | 3 + .../contexts/click_pipeline_context.py} | 2 +- .../pipeline_context.py} | 0 .../pipelines/{stolen => models}/singleton.py | 0 .../pipelines/pipelines/models/steps.py | 2 +- .../pipelines/stolen/lazy_decorator.py | 25 -------- .../test_steps/test_simple_docker_step.py | 2 +- 14 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py rename airbyte-ci/connectors/pipelines/pipelines/{stolen/base.py => models/contexts/click_pipeline_context.py} (98%) rename airbyte-ci/connectors/pipelines/pipelines/models/{contexts.py => contexts/pipeline_context.py} (100%) rename airbyte-ci/connectors/pipelines/pipelines/{stolen => models}/singleton.py (100%) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index db39c29b9655..18122e4fefb2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -19,7 +19,7 @@ from pipelines.helpers.github import update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook from pipelines.helpers.utils import METADATA_FILE_NAME -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext class ConnectorContext(PipelineContext): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 4cd43340e07a..42fbcedd4751 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -4,8 +4,8 @@ import click from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.stolen.base import ClickPipelineContext -from pipelines.stolen.lazy_decorator import LazyPassDecorator +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.cli.click_decorators import LazyPassDecorator pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @@ -18,21 +18,52 @@ def playground( ctx: ClickPipelineContext, ): - """Runs the tests for the given airbyte-ci package. + """ + TODO + 1. Make async + 1. Call a dagger pipeline - Args: - poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. - test_directory (str): The directory containing the tests to run. + Blockers: + 1. Need asyncio to run dagger pipeline """ - # ctx = ClickPipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) - # args = GlobalSettings(PLATFORM='Darwin') - # kwargs = {'opt': 'tight', 'hold': 'holdme'} + # dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") + # pytest_container = await ( + # dagger_client.container() + # .from_("python:3.10.12") + # .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + # .with_exec( + # sh_dash_c( + # [ + # "apt-get update", + # "apt-get install -y bash git curl", + # "pip install pipx", + # "pipx ensurepath", + # "pipx install poetry", + # ] + # ) + # ) + # .with_env_variable("VERSION", DOCKER_VERSION) + # .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + # .with_mounted_directory( + # "/airbyte", + # dagger_client.host().directory( + # ".", + # exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + # include=directories_to_mount, + # ), + # ) + # .with_workdir(f"/airbyte/{poetry_package_path}") + # .with_exec(["poetry", "install"]) + # .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + # .with_exec(["poetry", "run", "pytest", test_directory]) + # ) + + # await pytest_container + # return True + + print(f"params: {ctx.params}") - # (Pdb) ctx._click_context().args - # [] - # (Pdb) ctx._click_context().params - # {'opt': 'tight', 'hold': 'holdme'} - # (Pdb) + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py index 1dd5c76c2af2..964ee2c93b3f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -7,7 +7,7 @@ import dagger from pipelines.dagger.actions.python.pipx import with_installed_pipx_package from pipelines.dagger.containers.python import with_python_base -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import MountPath, Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 924fc8807fcf..d9def2b31873 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -11,7 +11,7 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py index 2a054c4df3dc..23e70650f405 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py index 3acaa28faff6..f7140f1e2ccd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py @@ -3,7 +3,7 @@ # from pipelines.dagger.actions.python.poetry import with_poetry_module -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index dcd6516c14da..532d37ebda15 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,8 +1,9 @@ import functools +from functools import wraps import inspect -from typing import Any, Callable +from typing import Any, Callable, Type import click @@ -85,3 +86,26 @@ def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper return decorator + + +class LazyPassDecorator: + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + @wraps(f) + def decorated_function(*args: Any, **kwargs: Any) -> Any: + # Check if the kwargs already contain the arguments being passed by the decorator + decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} + # Create an instance of the class + instance = self.cls(*self.args, **decorator_kwargs) + # If function has **kwargs, we can put the instance there + if 'kwargs' in kwargs: + kwargs['kwargs'] = instance + # Otherwise, add it to positional arguments + else: + args = (*args, instance) + return f(*args, **kwargs) + return decorated_function diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py similarity index 98% rename from airbyte-ci/connectors/pipelines/pipelines/stolen/base.py rename to airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index efcf51ed7bca..4cf262b4f3ae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -10,7 +10,7 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr -from .singleton import Singleton +from ..singleton import Singleton # this is a bit of a hack to get around how prefect resolves parameters diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/models/contexts.py rename to airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py rename to airbyte-ci/connectors/pipelines/pipelines/models/singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index 4c5f9a523da5..5cc119ed6440 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -21,7 +21,7 @@ from pipelines.helpers.utils import format_duration, get_exec_result if typing.TYPE_CHECKING: - from pipelines.models.contexts import PipelineContext + from pipelines.models.contexts.pipeline_context import PipelineContext from abc import ABC from typing import ClassVar diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py deleted file mode 100644 index b65e610c5212..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py +++ /dev/null @@ -1,25 +0,0 @@ -from functools import wraps -from typing import Any, Callable, Type - - -class LazyPassDecorator: - def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: - self.cls = cls - self.args = args - self.kwargs = kwargs - - def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: - @wraps(f) - def decorated_function(*args: Any, **kwargs: Any) -> Any: - # Check if the kwargs already contain the arguments being passed by the decorator - decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} - # Create an instance of the class - instance = self.cls(*self.args, **decorator_kwargs) - # If function has **kwargs, we can put the instance there - if 'kwargs' in kwargs: - kwargs['kwargs'] = instance - # Otherwise, add it to positional arguments - else: - args = (*args, instance) - return f(*args, **kwargs) - return decorated_function diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py index a4b23cbaea51..27d5765037fe 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -7,7 +7,7 @@ import pytest from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.helpers.utils import get_exec_result -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import MountPath pytestmark = [ From 8bfe2aa610a2121059b879876eed4ab3da2915b6 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Thu, 19 Oct 2023 21:48:50 +0000 Subject: [PATCH 009/141] Automated Commit - Formatting Changes --- .../airbyte_ci/playground/commands.py | 9 ++---- .../pipelines/pipelines/cli/airbyte_ci.py | 14 +++++++-- .../pipelines/cli/click_decorators.py | 31 +++++++------------ .../pipelines/pipelines/cli/telemetry.py | 1 + .../models/contexts/click_pipeline_context.py | 13 ++++---- .../pipelines/pipelines/models/singleton.py | 7 +++-- 6 files changed, 36 insertions(+), 39 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 42fbcedd4751..647a50cf6a20 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,13 +3,12 @@ # import click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.cli.click_decorators import LazyPassDecorator - pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.command() @click.argument("hold") @click.option("--opt", default="default_value") @@ -62,8 +61,4 @@ def playground( # await pytest_container # return True - - print(f"params: {ctx.params}") - - diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 2be63e30698b..f7026278fce6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -34,14 +34,16 @@ def display_welcome_message() -> None: - print(''' + print( + """ █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ - ''') + """ + ) def check_up_to_date() -> bool: @@ -147,6 +149,7 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) + def log_git_info(ctx: click.Context): main_logger.info("Running airbyte-ci in CI mode.") main_logger.info(f"CI Context: {ctx.obj['ci_context']}") @@ -159,11 +162,13 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") + def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: if ctx.obj["is_local"]: return "TESTEST" return ctx.obj["gha_workflow_run_id"] + def _get_pull_request(ctx: click.Context): pull_request_number = ctx.obj["pull_request_number"] ci_github_access_token = ctx.obj["ci_github_access_token"] @@ -174,6 +179,7 @@ def _get_pull_request(ctx: click.Context): return github.get_pull_request(pull_request_number, ci_github_access_token) + def _get_modified_files(ctx: click.Context) -> List[Path]: return transform_strs_to_paths( get_modified_files( @@ -182,12 +188,14 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: ctx.obj["diffed_branch"], ctx.obj["is_local"], ctx.obj["ci_context"], - ctx.obj["pull_request"] + ctx.obj["pull_request"], ) ) + # COMMANDS + @click.group( cls=LazyGroup, help="Airbyte CI top-level command group.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 532d37ebda15..47538c7a99de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,26 +1,18 @@ - - import functools -from functools import wraps import inspect +from functools import wraps from typing import Any, Callable, Type -import click +import click def _contains_var_kwarg(f): - return any( - param.kind == inspect.Parameter.VAR_KEYWORD - for param in inspect.signature(f).parameters.values() - ) + return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) def _is_kwarg_of(key, f): param = inspect.signature(f).parameters.get(key, False) - return param and ( - param.kind is inspect.Parameter.KEYWORD_ONLY or - param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - ) + return param and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) def click_ignore_unused_kwargs(f): @@ -36,12 +28,9 @@ def click_ignore_unused_kwargs(f): @functools.wraps(f) def inner(*args, **kwargs): - filtered_kwargs = { - key: value - for key, value in kwargs.items() - if _is_kwarg_of(key, f) - } + filtered_kwargs = {key: value for key, value in kwargs.items() if _is_kwarg_of(key, f)} return f(*args, **filtered_kwargs) + return inner @@ -69,6 +58,7 @@ def wrapper(*args, **kwargs): return wrapper + def click_append_to_context_object(key: str, value: Callable | Any): """ Decorator to append a value to the click context object. @@ -84,7 +74,9 @@ def wrapper(*args, **kwargs): else: ctx.obj[key] = value return f(*args, **kwargs) + return wrapper + return decorator @@ -102,10 +94,11 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any: # Create an instance of the class instance = self.cls(*self.args, **decorator_kwargs) # If function has **kwargs, we can put the instance there - if 'kwargs' in kwargs: - kwargs['kwargs'] = instance + if "kwargs" in kwargs: + kwargs["kwargs"] = instance # Otherwise, add it to positional arguments else: args = (*args, instance) return f(*args, **kwargs) + return decorated_function diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 942d69cb5751..891e0327e45a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -39,6 +39,7 @@ def click_track_command(f): """ Decorator to track CLI commands with segment.io """ + def wrapper(*args, **kwargs): ctx = get_current_context() top_level_command = ctx.command_path diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 4cf262b4f3ae..27c3725c36dd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -2,16 +2,15 @@ from typing import Any, Callable, Optional, Union import dagger - -# TODO ben set up for async -# from asyncclick import Context, get_current_context - from click import Context, get_current_context from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton +# TODO ben set up for async +# from asyncclick import Context, get_current_context + # this is a bit of a hack to get around how prefect resolves parameters # basically without this, prefect will attempt to access the context @@ -20,6 +19,7 @@ def get_context() -> Context: return get_current_context() + class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) @@ -41,7 +41,7 @@ def params(self): return {**click_obj, **click_params} class Config: - arbitrary_types_allowed=True + arbitrary_types_allowed = True def __init__(self, **data: dict[str, Any]): """ @@ -67,8 +67,7 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name async with self._dagger_client_lock: if not self._dagger_client: connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) - self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore client = self._dagger_client assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client - diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 3fb7f62aedbd..061307f1d016 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -11,10 +11,11 @@ class Singleton: The __new__ method ensures that only one instance of each subclass is created. The _initialized dictionary is used to control when the initialization logic of each subclass is executed. """ - _instances: dict[Type['Singleton'], Any] = {} - _initialized: dict[Type['Singleton'], bool] = {} - def __new__(cls: Type['Singleton'], *args: Any, **kwargs: Any) -> Any: + _instances: dict[Type["Singleton"], Any] = {} + _initialized: dict[Type["Singleton"], bool] = {} + + def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) From 27043ae2544608314beb929553fc739434216dc5 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 11:54:00 -0700 Subject: [PATCH 010/141] Bring up to date with async click --- airbyte-ci/connectors/pipelines/README.md | 9 +- .../airbyte_ci/playground/commands.py | 88 +++++++++++-------- .../pipelines/pipelines/cli/airbyte_ci.py | 11 ++- .../pipelines/cli/click_decorators.py | 2 +- .../pipelines/pipelines/cli/telemetry.py | 2 +- .../models/contexts/click_pipeline_context.py | 18 +--- .../connectors/pipelines/pyproject.toml | 2 +- 7 files changed, 71 insertions(+), 61 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index f833ba7e8a2d..54c47ca8a895 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -52,8 +52,8 @@ If you face any installation problems feel free to reach out the Airbyte Connect ### Setting up connector secrets access If you plan to use Airbyte CI to run CAT (Connector Acceptance Tests), we recommend setting up GSM -access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the -CI Credentials package (which Airbyte CI uses under the hood) README's +access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the +CI Credentials package (which Airbyte CI uses under the hood) README's [Get GSM Access](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access) instructions. @@ -155,7 +155,7 @@ Available commands: | `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. | | `--docker-hub-username` | | | DOCKER_HUB_USERNAME | Your username to connect to DockerHub. Required for the publish subcommand. | | `--docker-hub-password` | | | DOCKER_HUB_PASSWORD | Your password to connect to DockerHub. Required for the publish subcommand. | - + ### `connectors list` command Retrieve the list of connectors satisfying the provided filters. @@ -408,7 +408,8 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. +| 2.4.1 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | +| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | | 2.3.1 | [#31748](https://github.com/airbytehq/airbyte/pull/31748) | Use AsyncClick library instead of base Click. | | 2.3.0 | [#31699](https://github.com/airbytehq/airbyte/pull/31699) | Support optional concurrent CAT execution. | | 2.2.6 | [#31752](https://github.com/airbytehq/airbyte/pull/31752) | Only authenticate when secrets are available. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 647a50cf6a20..9ec67f6bd022 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -2,20 +2,25 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import click +import logging +import asyncclick as click from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.helpers.utils import sh_dash_c +from pipelines.consts import DOCKER_VERSION pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.argument("hold") -@click.option("--opt", default="default_value") +@click.argument("poetry_package_path") +@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") @pass_pipeline_context @click_ignore_unused_kwargs -def playground( +async def playground( ctx: ClickPipelineContext, + poetry_package_path: str, + test_directory: str, ): """ TODO @@ -25,40 +30,47 @@ def playground( Blockers: 1. Need asyncio to run dagger pipeline """ + print(f"params: {ctx.params}") - # dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") - # pytest_container = await ( - # dagger_client.container() - # .from_("python:3.10.12") - # .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - # .with_exec( - # sh_dash_c( - # [ - # "apt-get update", - # "apt-get install -y bash git curl", - # "pip install pipx", - # "pipx ensurepath", - # "pipx install poetry", - # ] - # ) - # ) - # .with_env_variable("VERSION", DOCKER_VERSION) - # .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - # .with_mounted_directory( - # "/airbyte", - # dagger_client.host().directory( - # ".", - # exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - # include=directories_to_mount, - # ), - # ) - # .with_workdir(f"/airbyte/{poetry_package_path}") - # .with_exec(["poetry", "install"]) - # .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - # .with_exec(["poetry", "run", "pytest", test_directory]) - # ) + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - # await pytest_container - # return True + dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") + pytest_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_exec(["poetry", "run", "pytest", test_directory]) + ) + + success = await pytest_container + if not success: + click.Abort() - print(f"params: {ctx.params}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 12f0c9521cef..77a18f6bf965 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -239,7 +239,6 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("modified_files", _get_modified_files) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -248,6 +247,16 @@ async def airbyte_ci( display_welcome_message() check_up_to_date() + modified_files = await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"], + ) + ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) + if not is_local: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 47538c7a99de..d4e849d97087 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -3,7 +3,7 @@ from functools import wraps from typing import Any, Callable, Type -import click +import asyncclick as click def _contains_var_kwarg(f): diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 891e0327e45a..7ad4bea8daaa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -9,7 +9,7 @@ import sys import segment.analytics as analytics -from click import get_current_context +from asyncclick import get_current_context analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" analytics.send = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 27c3725c36dd..2b6046cc9165 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,29 +1,17 @@ import sys -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional import dagger -from click import Context, get_current_context +from asyncclick import Context, get_current_context from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton -# TODO ben set up for async -# from asyncclick import Context, get_current_context - - -# this is a bit of a hack to get around how prefect resolves parameters -# basically without this, prefect will attempt to access the context -# before we create it in main.py in order to resolve it as a parameter -# wrapping it in a function like this prevents that from happening -def get_context() -> Context: - return get_current_context() - - class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) - _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) @property def params(self): diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index f0e28631d562..f09e10c53d96 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.4.0" +version = "2.4.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From c53310a85d4379feffdc2328c0bad6b62ca95907 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 12:20:41 -0700 Subject: [PATCH 011/141] Replace test with playground impl --- .../airbyte_ci/playground/__init__.py | 3 - .../airbyte_ci/playground/commands.py | 76 ------------------- .../pipelines/airbyte_ci/test/commands.py | 55 +++++++++++++- .../pipelines/airbyte_ci/test/pipeline.py | 71 ----------------- .../pipelines/pipelines/cli/airbyte_ci.py | 21 +++-- .../models/contexts/click_pipeline_context.py | 8 ++ 6 files changed, 75 insertions(+), 159 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py deleted file mode 100644 index c941b3045795..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py deleted file mode 100644 index 9ec67f6bd022..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -import asyncclick as click -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.helpers.utils import sh_dash_c -from pipelines.consts import DOCKER_VERSION - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - - -@click.command() -@click.argument("poetry_package_path") -@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") -@pass_pipeline_context -@click_ignore_unused_kwargs -async def playground( - ctx: ClickPipelineContext, - poetry_package_path: str, - test_directory: str, -): - """ - TODO - 1. Make async - 1. Call a dagger pipeline - - Blockers: - 1. Need asyncio to run dagger pipeline - """ - print(f"params: {ctx.params}") - - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - - dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_env_variable("VERSION", DOCKER_VERSION) - .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - - success = await pytest_container - if not success: - click.Abort() - diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index cf47f4168b5d..426dca551080 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -2,23 +2,74 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import asyncclick as click -from pipelines.airbyte_ci.test.pipeline import run_test +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.helpers.utils import sh_dash_c +from pipelines.consts import DOCKER_VERSION +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() @click.argument("poetry_package_path") @click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +@pass_pipeline_context +@click_ignore_unused_kwargs async def test( + ctx: ClickPipelineContext, poetry_package_path: str, test_directory: str, ): """Runs the tests for the given airbyte-ci package. Args: + ctx (ClickPipelineContext): The context object. poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ - success = await run_test(poetry_package_path, test_directory) + + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) + + pipeline_name = f"Unit tests for {poetry_package_path}" + dagger_client = await ctx.get_dagger_client(pipeline_name=pipeline_name) + pytest_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_exec(["poetry", "run", "pytest", test_directory]) + ) + + success = await pytest_container if not success: click.Abort() + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py deleted file mode 100644 index 16a8e322d7c6..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -import os -import sys - -import dagger -from pipelines.consts import DOCKER_VERSION -from pipelines.helpers.utils import sh_dash_c - - -async def run_test(poetry_package_path: str, test_directory: str) -> bool: - """Runs the tests for the given airbyte-ci package in a Dagger container. - - Args: - airbyte_ci_package_path (str): Path to the airbyte-ci package to test, relative to airbyte-ci directory. - Returns: - bool: True if the tests passed, False otherwise. - """ - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - docker_host_socket = dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_env_variable("VERSION", DOCKER_VERSION) - .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: - logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") - pytest_container = pytest_container.with_env_variable( - "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" - ).with_unix_socket("/var/run/buildkit/buildkitd.sock", docker_host_socket) - - await pytest_container - return True - except dagger.ExecError as e: - logger.error("Tests failed") - logger.error(e.stderr) - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 77a18f6bf965..4186d18c7669 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -36,12 +36,20 @@ def display_welcome_message() -> None: print( """ - █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ - ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ - ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ - ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ - ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ - ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ + ╔─────────────────────────────────────────────────────────────────────────────────────────────────╗ + │ │ + │ │ + │ /$$$$$$ /$$$$$$ /$$$$$$$ /$$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$ /$$$$$$ /$$$$$$ │ + │ /$$__ $$|_ $$_/| $$__ $$| $$__ $$| $$ /$$/|__ $$__/| $$_____/ /$$__ $$|_ $$_/ │ + │ | $$ \ $$ | $$ | $$ \ $$| $$ \ $$ \ $$ /$$/ | $$ | $$ | $$ \__/ | $$ │ + │ | $$$$$$$$ | $$ | $$$$$$$/| $$$$$$$ \ $$$$/ | $$ | $$$$$ /$$$$$$| $$ | $$ │ + │ | $$__ $$ | $$ | $$__ $$| $$__ $$ \ $$/ | $$ | $$__/|______/| $$ | $$ │ + │ | $$ | $$ | $$ | $$ \ $$| $$ \ $$ | $$ | $$ | $$ | $$ $$ | $$ │ + │ | $$ | $$ /$$$$$$| $$ | $$| $$$$$$$/ | $$ | $$ | $$$$$$$$ | $$$$$$/ /$$$$$$ │ + │ |__/ |__/|______/|__/ |__/|_______/ |__/ |__/ |________/ \______/ |______/ │ + │ │ + │ │ + ╚─────────────────────────────────────────────────────────────────────────────────────────────────╝ """ ) @@ -203,7 +211,6 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", - "playground": "pipelines.airbyte_ci.playground.commands.playground", }, ) @click.version_option(__installed_version__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 2b6046cc9165..1502f40ce13b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -55,6 +55,14 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name async with self._dagger_client_lock: if not self._dagger_client: connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) + + """ + Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. + + Caution: + Avoid using this client across multiple thread pools, as it can lead to errors. + Cross-thread pool calls are generally considered an anti-pattern. + """ self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore client = self._dagger_client assert client, "Error initializing Dagger client" From b07da000b60004ca7db0d1204d5cc118dea4e133 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 12:26:42 -0700 Subject: [PATCH 012/141] Add singleton test --- .../connectors/test/steps/common.py | 2 +- .../pipelines/pipelines/models/singleton.py | 1 - .../pipelines/tests/test_models/__init__.py | 3 +++ .../tests/test_models/test_singleton.py | 25 +++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 9995e7f047a8..13b52e67d51d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -20,7 +20,7 @@ from pipelines.dagger.actions import secrets from pipelines.dagger.containers import internal_tools from pipelines.helpers.utils import METADATA_FILE_NAME -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 061307f1d016..6c0ec3c74219 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -16,7 +16,6 @@ class Singleton: _initialized: dict[Type["Singleton"], bool] = {} def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: - if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) cls._initialized[cls] = False diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py new file mode 100644 index 000000000000..9bdb95e01dde --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -0,0 +1,25 @@ +from pipelines.models.singleton import Singleton + +class SingletonChild(Singleton): + def __init__(self): + if not self._initialized[self.__class__]: + self.value = "initialized" + self._initialized[self.__class__] = True + +def test_singleton_instance(): + instance1 = SingletonChild() + instance2 = SingletonChild() + assert instance1 is instance2 + +def test_singleton_unique_per_subclass(): + class AnotherSingletonChild(Singleton): + pass + + instance1 = SingletonChild() + instance2 = AnotherSingletonChild() + assert instance1 is not instance2 + +def test_singleton_initialized(): + instance = SingletonChild() + instance.value # This should initialize the instance + assert instance._initialized[SingletonChild] From 714d2019a35be134ed4bbafccf43a83514f571cd Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 30 Oct 2023 17:56:45 -0700 Subject: [PATCH 013/141] Fix low hanging comments --- .../airbyte_ci/connectors/commands.py | 26 ++++++---------- .../pipelines/pipelines/cli/airbyte_ci.py | 30 +++++++------------ .../pipelines/cli/click_decorators.py | 22 ++++++++++---- .../models/contexts/click_pipeline_context.py | 6 ++++ .../pipelines/pipelines/stolen/__init__.py | 3 -- 5 files changed, 41 insertions(+), 46 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 79c17ce6ecf9..dd798119ea29 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj, click_append_to_context_object from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -227,30 +227,22 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: envvar="DOCKER_HUB_PASSWORD", ) @click_merge_args_into_context_obj +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]) ) @click_ignore_unused_kwargs async def connectors( ctx: click.Context, - use_remote_secrets: Optional[bool], - names: Tuple[str], - languages: Tuple[ConnectorLanguage], - support_levels: Tuple[str], - modified: bool, - metadata_changes_only: bool, - metadata_query: str, - enable_dependency_scanning: bool, ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - ctx.obj["use_remote_secrets"] = should_use_remote_secrets(use_remote_secrets) ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( - names, - support_levels, - languages, - modified, - metadata_changes_only, - metadata_query, + ctx.obj["names"], + ctx.obj["support_levels"], + ctx.obj["languages"], + ctx.obj["modified"], + ctx.obj["metadata_changes_only"], + ctx.obj["metadata_query"], ctx.obj["modified_files"], - enable_dependency_scanning, + ctx.obj["enable_dependency_scanning"], ) log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 9723580f614a..48c98545e963 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -173,13 +173,15 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") -def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: - if ctx.obj["is_local"]: - return "TESTEST" - return ctx.obj["gha_workflow_run_id"] +def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: + gha_workflow_run_id = ctx.obj["gha_workflow_run_id"] + if not gha_workflow_run_id: + return None + + return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" -def _get_pull_request(ctx: click.Context): +def _get_pull_request(ctx: click.Context) -> PullRequest or None: pull_request_number = ctx.obj["pull_request_number"] ci_github_access_token = ctx.obj["ci_github_access_token"] @@ -190,17 +192,6 @@ def _get_pull_request(ctx: click.Context): return github.get_pull_request(pull_request_number, ci_github_access_token) -def _get_modified_files(ctx: click.Context) -> List[Path]: - return transform_strs_to_paths( - get_modified_files( - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj["diffed_branch"], - ctx.obj["is_local"], - ctx.obj["ci_context"], - ctx.obj["pull_request"], - ) - ) def check_local_docker_configuration(): try: docker_client = docker.from_env() @@ -258,15 +249,14 @@ def check_local_docker_configuration(): @click_track_command @click_merge_args_into_context_obj @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) -@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) +@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, - is_local: bool, ): # noqa D103 display_welcome_message() - if is_local: + if ctx.obj["is_local"]: # This check is meaningful only when running locally # In our CI the docker host used by the Dagger Engine is different from the one used by the runner. check_local_docker_configuration() @@ -283,7 +273,7 @@ async def airbyte_ci( ) ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) - if not is_local: + if not ctx.obj["is_local"]: log_git_info(ctx) set_working_directory_to_root() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index d4e849d97087..0857e5559b4a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -6,16 +6,16 @@ import asyncclick as click -def _contains_var_kwarg(f): - return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) +def _contains_var_kwarg(f: Callable) -> bool: + return any(param.kind is inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) -def _is_kwarg_of(key, f): +def _is_kwarg_of(key: str, f: Callable) -> bool: param = inspect.signature(f).parameters.get(key, False) return param and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) -def click_ignore_unused_kwargs(f): +def click_ignore_unused_kwargs(f: Callable) -> Callable: """Make function ignore unmatched kwargs. If the function already has the catch all **kwargs, do nothing. @@ -34,7 +34,7 @@ def inner(*args, **kwargs): return inner -def click_merge_args_into_context_obj(f): +def click_merge_args_into_context_obj(f: Callable) -> Callable: """ Decorator to pass click context and args to children commands. """ @@ -59,7 +59,7 @@ def wrapper(*args, **kwargs): return wrapper -def click_append_to_context_object(key: str, value: Callable | Any): +def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ @@ -81,12 +81,22 @@ def wrapper(*args, **kwargs): class LazyPassDecorator: + """ + Used to create a decorator that will pass an instance of the given class to the decorated function. + """ def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + """ + Initialize the decorator with the given source class + """ self.cls = cls self.args = args self.kwargs = kwargs def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + """ + Create a decorator that will pass an instance of the given class to the decorated function. + """ + @wraps(f) def decorated_function(*args: Any, **kwargs: Any) -> Any: # Check if the kwargs already contain the arguments being passed by the decorator diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 1502f40ce13b..7e6dec040050 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -9,6 +9,12 @@ from ..singleton import Singleton class ClickPipelineContext(BaseModel, Singleton): + """ + A replacement class for the Click context object passed to click functions. + + This class is meant to serve as a singleton object that initializes and holds onto a single instance of the + Dagger client, which is used to create containers for running pipelines. + """ dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py deleted file mode 100644 index c941b3045795..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# From 29087ea62f42e781ddabb72d8f92a8963cd87832 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Tue, 31 Oct 2023 01:09:24 +0000 Subject: [PATCH 014/141] Automated Commit - Formatting Changes --- .../pipelines/pipelines/airbyte_ci/connectors/commands.py | 4 ++-- .../pipelines/pipelines/airbyte_ci/test/commands.py | 7 ++++--- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 1 + .../connectors/pipelines/pipelines/cli/click_decorators.py | 1 + .../pipelines/models/contexts/click_pipeline_context.py | 2 ++ .../pipelines/tests/test_models/test_singleton.py | 4 ++++ 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index dd798119ea29..27c7724796e9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj, click_append_to_context_object +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -227,7 +227,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: envvar="DOCKER_HUB_PASSWORD", ) @click_merge_args_into_context_obj -@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]) ) +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) @click_ignore_unused_kwargs async def connectors( ctx: click.Context, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 426dca551080..f96330a27587 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -3,14 +3,16 @@ # import logging + import asyncclick as click from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.helpers.utils import sh_dash_c from pipelines.consts import DOCKER_VERSION +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.command() @click.argument("poetry_package_path") @click.option("--test-directory", default="tests", help="The directory containing the tests to run.") @@ -72,4 +74,3 @@ async def test( success = await pytest_container if not success: click.Abort() - diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 48c98545e963..a5f0d23a6067 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -276,6 +276,7 @@ async def airbyte_ci( if not ctx.obj["is_local"]: log_git_info(ctx) + set_working_directory_to_root() if __name__ == "__main__": diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 0857e5559b4a..88cb0019714a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -84,6 +84,7 @@ class LazyPassDecorator: """ Used to create a decorator that will pass an instance of the given class to the decorated function. """ + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: """ Initialize the decorator with the given source class diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 7e6dec040050..68c26c39fb91 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -8,6 +8,7 @@ from ..singleton import Singleton + class ClickPipelineContext(BaseModel, Singleton): """ A replacement class for the Click context object passed to click functions. @@ -15,6 +16,7 @@ class ClickPipelineContext(BaseModel, Singleton): This class is meant to serve as a singleton object that initializes and holds onto a single instance of the Dagger client, which is used to create containers for running pipelines. """ + dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py index 9bdb95e01dde..2a5b27998ff1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -1,16 +1,19 @@ from pipelines.models.singleton import Singleton + class SingletonChild(Singleton): def __init__(self): if not self._initialized[self.__class__]: self.value = "initialized" self._initialized[self.__class__] = True + def test_singleton_instance(): instance1 = SingletonChild() instance2 = SingletonChild() assert instance1 is instance2 + def test_singleton_unique_per_subclass(): class AnotherSingletonChild(Singleton): pass @@ -19,6 +22,7 @@ class AnotherSingletonChild(Singleton): instance2 = AnotherSingletonChild() assert instance1 is not instance2 + def test_singleton_initialized(): instance = SingletonChild() instance.value # This should initialize the instance From fb43cfb97e7154c0fa3cfcdbeb5f290645864cf7 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 31 Oct 2023 16:49:29 -0700 Subject: [PATCH 015/141] Move decorator to file --- .../pipelines/pipelines/airbyte_ci/test/commands.py | 10 +++------- airbyte-ci/connectors/pipelines/pipelines/consts.py | 1 + .../models/contexts/click_pipeline_context.py | 6 ++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f96330a27587..be40f0cace80 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -5,12 +5,10 @@ import logging import asyncclick as click -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() @@ -71,6 +69,4 @@ async def test( .with_exec(["poetry", "run", "pytest", test_directory]) ) - success = await pytest_container - if not success: - click.Abort() + await pytest_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 7b4805eabb51..5d4a879ba4fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -27,6 +27,7 @@ PLATFORM_MACHINE_TO_DAGGER_PLATFORM = { "x86_64": Platform("linux/amd64"), + "aarch64": Platform("linux/amd64"), "arm64": Platform("linux/arm64"), "amd64": Platform("linux/amd64"), } diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 68c26c39fb91..69d45157d946 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -6,9 +6,12 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr +from pipelines.cli.click_decorators import LazyPassDecorator + from ..singleton import Singleton + class ClickPipelineContext(BaseModel, Singleton): """ A replacement class for the Click context object passed to click functions. @@ -75,3 +78,6 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name client = self._dagger_client assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client + +# Create @pass_pipeline_context decorator for use in click commands +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) From 323139d81765ebc2872a2eb880baeb01665627a3 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 31 Oct 2023 18:07:02 -0700 Subject: [PATCH 016/141] Add tests --- .../pipelines/models/contexts/click_pipeline_context.py | 5 ++--- airbyte-ci/connectors/pipelines/tests/conftest.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 69d45157d946..ad1ca92cf866 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -10,6 +10,7 @@ from ..singleton import Singleton +import anyio class ClickPipelineContext(BaseModel, Singleton): @@ -57,9 +58,7 @@ def __init__(self, **data: dict[str, Any]): super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True - import asyncio - - _dagger_client_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock) + _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name: Optional[str] = None) -> Client: if not self._dagger_client: diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 318e05aa5f86..17a4e1625189 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -23,8 +23,12 @@ def anyio_backend(): @pytest.fixture(scope="module") -async def dagger_client(): - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: +def dagger_connection(): + return dagger.Connection(dagger.Config(log_output=sys.stderr)) + +@pytest.fixture(scope="module") +async def dagger_client(dagger_connection): + async with dagger_connection as client: yield client From f624fa340ba6c218d563547de0ad99bad1149211 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 13:10:13 -0700 Subject: [PATCH 017/141] Add forgotten test --- .../test_click_pipeline_context.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py new file mode 100644 index 000000000000..8d9e3cb18848 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -0,0 +1,64 @@ +import asyncclick as click +import dagger +import pytest +from unittest.mock import patch +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +@pytest.mark.anyio +async def test_get_dagger_client_singleton(dagger_connection): + @click.command() + def cli(): + pass + + ctx = click.Context(cli) + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + client1 = await click_pipeline_context.get_dagger_client() + client2 = await click_pipeline_context.get_dagger_client() + client3 = await click_pipeline_context.get_dagger_client(pipeline_name="pipeline_name") + assert isinstance(client1, dagger.Client) + assert isinstance(client2, dagger.Client) + assert isinstance(client3, dagger.Client) + + assert client1 == client2 + assert client1 != client3 + +@pytest.mark.anyio +async def test_get_dagger_client_click_params(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"baz": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + pipeline_context_params = click_pipeline_context.params + assert pipeline_context_params == {**given_click_obj, **given_click_params} + +@pytest.mark.anyio +async def test_get_dagger_client_click_params_duplicate(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"foo": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + ctx.command.params = [click.Option(["--foo"])] + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with pytest.raises(ValueError): + click_pipeline_context.params + From 24297dff8968c715e4c042de5e49800fabbadc9b Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 13:51:16 -0700 Subject: [PATCH 018/141] Update ctx to click pipeline context --- .../pipelines/pipelines/airbyte_ci/test/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index be40f0cace80..e73d3baf695f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -17,14 +17,14 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test( - ctx: ClickPipelineContext, + pipeline_context: ClickPipelineContext, poetry_package_path: str, test_directory: str, ): """Runs the tests for the given airbyte-ci package. Args: - ctx (ClickPipelineContext): The context object. + pipeline_context (ClickPipelineContext): The context object. poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ @@ -37,7 +37,7 @@ async def test( directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) pipeline_name = f"Unit tests for {poetry_package_path}" - dagger_client = await ctx.get_dagger_client(pipeline_name=pipeline_name) + dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name) pytest_container = await ( dagger_client.container() .from_("python:3.10.12") From 58b28af893e9e71b35d699f46dd5a9aa689c1a9d Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:23:57 -0600 Subject: [PATCH 019/141] Add cli tests --- .../pipelines/pipelines/cli/airbyte_ci.py | 22 ++++--- .../pipelines/cli/click_decorators.py | 4 +- .../pipelines/tests/test_cli/__init__.py | 3 + .../tests/test_cli/test_click_decorators.py | 64 +++++++++++++++++++ 4 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index a5f0d23a6067..bb4eacddacb3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -206,6 +206,17 @@ def check_local_docker_configuration(): ) +async def get_modified_files_str(ctx: click.Context): + modified_files = await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"], + ) + return transform_strs_to_paths(modified_files) + # COMMANDS @@ -251,6 +262,7 @@ def check_local_docker_configuration(): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) +@click_append_to_context_object("modified_files", get_modified_files_str) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -263,16 +275,6 @@ async def airbyte_ci( check_up_to_date() - modified_files = await get_modified_files( - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj["diffed_branch"], - ctx.obj["is_local"], - ctx.obj["ci_context"], - ctx.obj["pull_request"], - ) - ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) - if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 88cb0019714a..ab1dac128c66 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -65,12 +65,14 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ def decorator(f): - def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) if callable(value): ctx.obj[key] = value(ctx) + elif inspect.iscoroutinefunction(value): + ctx.obj[key] = await value(ctx) else: ctx.obj[key] = value return f(*args, **kwargs) diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py new file mode 100644 index 000000000000..6ce6252ee9f2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -0,0 +1,64 @@ +import pytest +import asyncclick as click +from asyncclick.testing import CliRunner +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj + +@pytest.mark.anyio +async def test_click_append_to_context_object(): + runner = CliRunner() + + def get_value(ctx): + return "got" + + async def get_async_value(ctx): + return "async_got" + + @click.command(name='test-command') + @click_append_to_context_object('foo', 'bar') + @click_append_to_context_object('baz', lambda _ctx: "qux") + @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + @click_append_to_context_object('get', get_value) + @click_append_to_context_object('async_get', get_async_value) + + def test_command(): + ctx = click.get_current_context() + assert ctx.obj['foo'] == 'bar' + assert ctx.obj['baz'] == 'qux' + assert ctx.obj['foo2'] == 'bar2' + assert ctx.obj['get'] == 'got' + assert ctx.obj['async_get'] == 'async_got' + + + result = await runner.invoke(test_command) + assert result.exit_code == 0 + +@pytest.mark.anyio +async def test_click_ignore_unused_kwargs(): + @click_ignore_unused_kwargs + def decorated_function(a, b): + return a + b + + # Test that the decorated function works as expected with matching kwargs + assert decorated_function(a=1, b=2) == 3 + + # Test that the decorated function ignores unmatched kwargs + assert decorated_function(a=1, b=2, c=3) == 3 + + +@pytest.mark.anyio +async def test_click_merge_args_into_context_obj(): + runner = CliRunner() + + @click.command(name='test-command') + @click.option('--foo', help='foo option') + @click.option('--bar', help='bar option') + @click_merge_args_into_context_obj + @click_ignore_unused_kwargs + def test_command(ctx, foo, bar): + ctx = click.get_current_context() + assert ctx.obj['foo'] == foo + assert ctx.obj['bar'] == bar + + result = await runner.invoke(test_command, ['--foo', 'hello', '--bar', 'world']) + assert result.exit_code == 0 + From a4e1efb3818a2804fca7c380ac0b6b8fbfc6c19e Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:41:14 -0600 Subject: [PATCH 020/141] Remove async --- .../pipelines/pipelines/airbyte_ci/connectors/commands.py | 2 +- .../pipelines/airbyte_ci/connectors/list/commands.py | 1 + airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py | 4 +++- .../connectors/pipelines/pipelines/cli/click_decorators.py | 4 +--- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 27c7724796e9..be478a16b180 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -234,7 +234,7 @@ async def connectors( ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - + import pdb; pdb.set_trace() ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( ctx.obj["names"], ctx.obj["support_levels"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 4b0f54e6c462..c9945af1ef52 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -14,6 +14,7 @@ async def list( ctx: click.Context, ): + import pdb; pdb.set_trace() selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) table = Table(title=f"{len(selected_connectors)} selected connectors") table.add_column("Modified") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index bb4eacddacb3..5c5c4e336c7b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -262,7 +262,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("modified_files", get_modified_files_str) +@click_append_to_context_object("testme", "OUTTT") @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -275,6 +275,8 @@ async def airbyte_ci( check_up_to_date() + ctx.obj["modified_files"] = await get_modified_files_str(ctx) + if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index ab1dac128c66..88cb0019714a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -65,14 +65,12 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ def decorator(f): - async def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) if callable(value): ctx.obj[key] = value(ctx) - elif inspect.iscoroutinefunction(value): - ctx.obj[key] = await value(ctx) else: ctx.obj[key] = value return f(*args, **kwargs) From b1e479b3706322587a5dd68148e2d892efdf0c66 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:58:42 -0600 Subject: [PATCH 021/141] Add async back --- .../airbyte_ci/connectors/commands.py | 2 +- .../airbyte_ci/connectors/list/commands.py | 1 - .../pipelines/pipelines/cli/airbyte_ci.py | 4 ++-- .../pipelines/cli/click_decorators.py | 11 ++++++---- .../tests/test_cli/test_click_decorators.py | 20 ++++++++++++++++--- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index be478a16b180..27c7724796e9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -234,7 +234,7 @@ async def connectors( ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - import pdb; pdb.set_trace() + ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( ctx.obj["names"], ctx.obj["support_levels"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index c9945af1ef52..4b0f54e6c462 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -14,7 +14,6 @@ async def list( ctx: click.Context, ): - import pdb; pdb.set_trace() selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) table = Table(title=f"{len(selected_connectors)} selected connectors") table.add_column("Modified") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 5c5c4e336c7b..c6ef0b663ad1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -262,7 +262,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("testme", "OUTTT") +@click_append_to_context_object("modified_files", get_modified_files_str) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -275,7 +275,7 @@ async def airbyte_ci( check_up_to_date() - ctx.obj["modified_files"] = await get_modified_files_str(ctx) + # ctx.obj["modified_files"] = await get_modified_files_str(ctx) if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 88cb0019714a..cb1cd7da6e9e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -63,17 +63,20 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ - def decorator(f): - def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) - if callable(value): + + # if async, get the value, cannot use await + if inspect.iscoroutinefunction(value): + ctx.obj[key] = await value(ctx) + elif callable(value): ctx.obj[key] = value(ctx) else: ctx.obj[key] = value - return f(*args, **kwargs) + return await f(*args, **kwargs) return wrapper diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 6ce6252ee9f2..64cea33cced6 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -14,14 +14,25 @@ async def get_async_value(ctx): return "async_got" @click.command(name='test-command') + @click_append_to_context_object('get', get_value) + @click_append_to_context_object('async_get', get_async_value) @click_append_to_context_object('foo', 'bar') @click_append_to_context_object('baz', lambda _ctx: "qux") @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + def test_command(ctx): + assert ctx.obj['foo'] == 'bar' + assert ctx.obj['baz'] == 'qux' + assert ctx.obj['foo2'] == 'bar2' + assert ctx.obj['get'] == 'got' + assert ctx.obj['async_get'] == 'async_got' + + @click.command(name='test-command') @click_append_to_context_object('get', get_value) @click_append_to_context_object('async_get', get_async_value) - - def test_command(): - ctx = click.get_current_context() + @click_append_to_context_object('foo', 'bar') + @click_append_to_context_object('baz', lambda _ctx: "qux") + @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + async def test_command_async(ctx): assert ctx.obj['foo'] == 'bar' assert ctx.obj['baz'] == 'qux' assert ctx.obj['foo2'] == 'bar2' @@ -32,6 +43,9 @@ def test_command(): result = await runner.invoke(test_command) assert result.exit_code == 0 + result_async = await runner.invoke(test_command_async) + assert result_async.exit_code == 0 + @pytest.mark.anyio async def test_click_ignore_unused_kwargs(): @click_ignore_unused_kwargs From d89f7f38c225d75d83dfd01603793db83b2a2f1b Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Nov 2023 09:15:06 -0600 Subject: [PATCH 022/141] Add doc string --- .../pipelines/models/contexts/click_pipeline_context.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index ad1ca92cf866..a8abda48fb89 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -27,6 +27,11 @@ class ClickPipelineContext(BaseModel, Singleton): @property def params(self): + """ + Returns a combination of the click context object and the click context params. + + This means that any arguments or options defined in the parent command will be available to the child command. + """ ctx = self._click_context() click_obj = ctx.obj click_params = ctx.params From 7ca02fcfaf5f9ae9d048abf7e07ae14986e001a8 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Thu, 2 Nov 2023 15:31:46 +0000 Subject: [PATCH 023/141] Automated Commit - Formatting Changes --- .../pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/cli/click_decorators.py | 2 +- .../models/contexts/click_pipeline_context.py | 7 +- .../connectors/pipelines/tests/conftest.py | 1 + .../tests/test_cli/test_click_decorators.py | 64 +++++++++---------- .../test_click_pipeline_context.py | 13 ++-- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index c6ef0b663ad1..a4a9147e9ac8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -217,6 +217,7 @@ async def get_modified_files_str(ctx: click.Context): ) return transform_strs_to_paths(modified_files) + # COMMANDS diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index cb1cd7da6e9e..213c28cb5bbc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -63,12 +63,12 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ + def decorator(f): async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) - # if async, get the value, cannot use await if inspect.iscoroutinefunction(value): ctx.obj[key] = await value(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index a8abda48fb89..6895aacc166b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,17 +1,15 @@ import sys from typing import Any, Callable, Optional +import anyio import dagger from asyncclick import Context, get_current_context from dagger.api.gen import Client, Container -from pydantic import BaseModel, Field, PrivateAttr - from pipelines.cli.click_decorators import LazyPassDecorator +from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton -import anyio - class ClickPipelineContext(BaseModel, Singleton): """ @@ -83,5 +81,6 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client + # Create @pass_pipeline_context decorator for use in click commands pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 17a4e1625189..fe7314bbebe7 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -26,6 +26,7 @@ def anyio_backend(): def dagger_connection(): return dagger.Connection(dagger.Config(log_output=sys.stderr)) + @pytest.fixture(scope="module") async def dagger_client(dagger_connection): async with dagger_connection as client: diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 64cea33cced6..0aaf7d85cd48 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -1,8 +1,9 @@ -import pytest import asyncclick as click +import pytest from asyncclick.testing import CliRunner from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj + @pytest.mark.anyio async def test_click_append_to_context_object(): runner = CliRunner() @@ -13,32 +14,31 @@ def get_value(ctx): async def get_async_value(ctx): return "async_got" - @click.command(name='test-command') - @click_append_to_context_object('get', get_value) - @click_append_to_context_object('async_get', get_async_value) - @click_append_to_context_object('foo', 'bar') - @click_append_to_context_object('baz', lambda _ctx: "qux") - @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") def test_command(ctx): - assert ctx.obj['foo'] == 'bar' - assert ctx.obj['baz'] == 'qux' - assert ctx.obj['foo2'] == 'bar2' - assert ctx.obj['get'] == 'got' - assert ctx.obj['async_get'] == 'async_got' - - @click.command(name='test-command') - @click_append_to_context_object('get', get_value) - @click_append_to_context_object('async_get', get_async_value) - @click_append_to_context_object('foo', 'bar') - @click_append_to_context_object('baz', lambda _ctx: "qux") - @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" + + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") async def test_command_async(ctx): - assert ctx.obj['foo'] == 'bar' - assert ctx.obj['baz'] == 'qux' - assert ctx.obj['foo2'] == 'bar2' - assert ctx.obj['get'] == 'got' - assert ctx.obj['async_get'] == 'async_got' - + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" result = await runner.invoke(test_command) assert result.exit_code == 0 @@ -46,6 +46,7 @@ async def test_command_async(ctx): result_async = await runner.invoke(test_command_async) assert result_async.exit_code == 0 + @pytest.mark.anyio async def test_click_ignore_unused_kwargs(): @click_ignore_unused_kwargs @@ -63,16 +64,15 @@ def decorated_function(a, b): async def test_click_merge_args_into_context_obj(): runner = CliRunner() - @click.command(name='test-command') - @click.option('--foo', help='foo option') - @click.option('--bar', help='bar option') + @click.command(name="test-command") + @click.option("--foo", help="foo option") + @click.option("--bar", help="bar option") @click_merge_args_into_context_obj @click_ignore_unused_kwargs def test_command(ctx, foo, bar): ctx = click.get_current_context() - assert ctx.obj['foo'] == foo - assert ctx.obj['bar'] == bar + assert ctx.obj["foo"] == foo + assert ctx.obj["bar"] == bar - result = await runner.invoke(test_command, ['--foo', 'hello', '--bar', 'world']) + result = await runner.invoke(test_command, ["--foo", "hello", "--bar", "world"]) assert result.exit_code == 0 - diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index 8d9e3cb18848..c1b8959ad017 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -1,9 +1,11 @@ +from unittest.mock import patch + import asyncclick as click import dagger import pytest -from unittest.mock import patch from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + @pytest.mark.anyio async def test_get_dagger_client_singleton(dagger_connection): @click.command() @@ -14,7 +16,7 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): client1 = await click_pipeline_context.get_dagger_client() client2 = await click_pipeline_context.get_dagger_client() client3 = await click_pipeline_context.get_dagger_client(pipeline_name="pipeline_name") @@ -25,6 +27,7 @@ def cli(): assert client1 == client2 assert client1 != client3 + @pytest.mark.anyio async def test_get_dagger_client_click_params(dagger_connection): @click.command() @@ -39,10 +42,11 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): pipeline_context_params = click_pipeline_context.params assert pipeline_context_params == {**given_click_obj, **given_click_params} + @pytest.mark.anyio async def test_get_dagger_client_click_params_duplicate(dagger_connection): @click.command() @@ -58,7 +62,6 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): with pytest.raises(ValueError): click_pipeline_context.params - From 8fb3ba69ed116bfb660ce92eb5113145ba730c1e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 16:08:23 -0500 Subject: [PATCH 024/141] Run license check on repo --- airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py | 2 ++ .../connectors/pipelines/pipelines/cli/click_decorators.py | 2 ++ airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py | 2 ++ .../pipelines/models/contexts/click_pipeline_context.py | 2 ++ airbyte-ci/connectors/pipelines/pipelines/models/singleton.py | 2 ++ .../pipelines/tests/test_cli/test_click_decorators.py | 2 ++ .../pipelines/tests/test_models/test_click_pipeline_context.py | 2 ++ .../connectors/pipelines/tests/test_models/test_singleton.py | 2 ++ .../source_azure_blob_storage/stream_reader.py | 2 ++ .../source-azure-blob-storage/unit_tests/unit_tests.py | 2 ++ .../connectors/source-github/unit_tests/conftest.py | 2 ++ .../connectors/source-linkedin-ads/unit_tests/conftest.py | 2 ++ .../connectors/source-s3/source_s3/v4/zip_reader.py | 2 ++ .../connectors/source-s3/unit_tests/v4/test_zip_reader.py | 2 ++ .../connectors/source-tiktok-marketing/unit_tests/conftest.py | 2 ++ .../connectors/source-zendesk-support/unit_tests/conftest.py | 2 ++ 16 files changed, 32 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py index 7a22c221d579..54c3e984f93c 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import time from typing import Any, Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 213c28cb5bbc..2c4843b973ba 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import functools import inspect from functools import wraps diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py index c76c0a7c5477..b7214a26f7f9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + # Source: https://click.palletsprojects.com/en/8.1.x/complex/ import importlib diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 6895aacc166b..1d56ce91e421 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import sys from typing import Any, Callable, Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 6c0ec3c74219..e9d759d401f6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Type diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 0aaf7d85cd48..c70b2350147b 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import asyncclick as click import pytest from asyncclick.testing import CliRunner diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index c1b8959ad017..9a685372c6cd 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from unittest.mock import patch import asyncclick as click diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py index 2a5b27998ff1..87648f432a12 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from pipelines.models.singleton import Singleton diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index 0f6ff2081174..47a235c00f14 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging from contextlib import contextmanager from io import IOBase diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py index 17b5f7fc5913..88d003f81475 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from source_azure_blob_storage.legacy_config_transformer import LegacyConfigTransformer diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 8168c53508c6..4f475b80c797 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import io import struct import zipfile diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py index 8120fe52e434..c97e468cba6d 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import datetime import io import struct diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index a2c4975260d2..969410ee24e1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os import pytest diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" From 861d94bd90ecb42a20680437bdff07476edb216d Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 23 Oct 2023 16:15:12 -0500 Subject: [PATCH 025/141] add check command for python --- .../pipelines/airbyte_ci/check/commands.py | 80 +++++++++++++++++++ .../pipelines/pipelines/cli/airbyte_ci.py | 1 + 2 files changed, 81 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py new file mode 100644 index 000000000000..3cb1369ad02a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the tests command to test airbyte-ci projects. +""" + +import logging +import sys + +import asyncclick as click +import dagger +from pipelines.consts import DOCKER_VERSION +from pipelines.helpers.utils import sh_dash_c + + +@click.command() +@click.option("--fix", is_flag=True, default=False, help="Fix any formatting issues detected.") +async def check(fix: bool): + """Checks whether the repository is formatted correctly.""" + success = await run_check(fix) + if not success: + click.Abort() + + +async def run_check(fix: bool) -> bool: + """Checks whether the repository is formatted correctly. + Args: + fix (bool): Whether to automatically fix any formatting issues detected. + Returns: + bool: True if the check/format succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + isort_command = ["poetry", "run", 'isort', '--settings-file', "pyproject.toml", '--check-only', "."] + black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] + if fix: + isort_command.remove("--check-only") + black_command.remove("--check") + + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + format_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.py", "pyproject.toml", "poetry.lock"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build"] + ) + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(isort_command) + .with_exec(black_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index a4a9147e9ac8..328110567c62 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -226,6 +226,7 @@ async def get_modified_files_str(ctx: click.Context): help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "check": "pipelines.airbyte_ci.check.commands.check", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", }, From 584cfdfdc79a44c422dbfc23b4a19d59786020b4 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 23 Oct 2023 16:22:51 -0500 Subject: [PATCH 026/141] add poetry and pyproject files, ignore git --- .../pipelines/airbyte_ci/check/commands.py | 2 +- poetry.lock | 140 ++++++++++++++++++ pyproject.toml | 13 ++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 poetry.lock diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py index 3cb1369ad02a..46bbef36fe03 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py @@ -61,7 +61,7 @@ async def run_check(fix: bool) -> bool: dagger_client.host().directory( ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build"] + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] ) ) .with_workdir(f"/src") diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000000..b2f8a71f7ef2 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,140 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "black" +version = "22.3.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "isort" +version = "5.6.4" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "isort-5.6.4-py3-none-any.whl", hash = "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7"}, + {file = "isort-5.6.4.tar.gz", hash = "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "pathspec" +version = "0.11.2" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, +] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "~3.10" +content-hash = "f28ec219b6bb87ac12bec09b508639435f6468a2bdf1190552a55b669bf1eee2" diff --git a/pyproject.toml b/pyproject.toml index 80a70a5bd99e..6625366b1fa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,16 @@ +[tool.poetry] +name = "airbyte" +version = "0.1.0" +description = "Airbyte open source connector code" +authors = ["Airbyte "] + +[tool.poetry.dependencies] +python = "~3.10" +black = "~22.3.0" + +[tool.poetry.group.dev.dependencies] +isort = "5.6.4" + [tool.black] line-length = 140 target-version = ["py37"] From d0db4095ca353f1e8ae95f65da4ccfaed3b3993f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:09:09 -0500 Subject: [PATCH 027/141] make black a dev dependency --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index b2f8a71f7ef2..7e3ee153c35a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -137,4 +137,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "f28ec219b6bb87ac12bec09b508639435f6468a2bdf1190552a55b669bf1eee2" +content-hash = "352be223e781ec8ab7dc7326b50ba69733b74792f65832f00f185a102785caf4" diff --git a/pyproject.toml b/pyproject.toml index 6625366b1fa8..fb3ac8175446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,10 +6,10 @@ authors = ["Airbyte "] [tool.poetry.dependencies] python = "~3.10" -black = "~22.3.0" [tool.poetry.group.dev.dependencies] isort = "5.6.4" +black = "~22.3.0" [tool.black] line-length = 140 From 6df5f5813c14d0dbf6b921e354bbf7178d5096ab Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:10:58 -0500 Subject: [PATCH 028/141] format new code --- .../pipelines/pipelines/airbyte_ci/check/commands.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py index 46bbef36fe03..8e87590f124f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py @@ -32,12 +32,11 @@ async def run_check(fix: bool) -> bool: bool: True if the check/format succeeded, false otherwise """ logger = logging.getLogger(f"format") - isort_command = ["poetry", "run", 'isort', '--settings-file', "pyproject.toml", '--check-only', "."] + isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] if fix: isort_command.remove("--check-only") black_command.remove("--check") - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: try: @@ -59,10 +58,10 @@ async def run_check(fix: bool) -> bool: .with_mounted_directory( "/src", dagger_client.host().directory( - ".", - include=["**/*.py", "pyproject.toml", "poetry.lock"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] - ) + ".", + include=["**/*.py", "pyproject.toml", "poetry.lock"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"], + ), ) .with_workdir(f"/src") .with_exec(["poetry", "install"]) From 6449446e02408d415fba9cdd70f4c98f64b60b4a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:14:18 -0500 Subject: [PATCH 029/141] remove black and isort code from build.gradle --- build.gradle | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/build.gradle b/build.gradle index 119ec338cd1e..c85b7f324c66 100644 --- a/build.gradle +++ b/build.gradle @@ -301,44 +301,6 @@ allprojects { licensePythonFormat.configure { dependsOn tasks.matching { it.name == 'generate' } } - - def registerPythonFormatTask = { - String taskName, String moduleName, String settingsFileFlagName, String... extraArgs -> - def task = tasks.register(taskName, Exec) { - workingDir projectDir - commandLine rootProject.file('.venv/bin/python') - args "-m", moduleName, settingsFileFlagName, rootProject.file('pyproject.toml').absolutePath - args extraArgs - pythonTarget.getFiles().forEach { - args projectDir.relativePath(it) - } - } - task.configure { - dependsOn rootProject.tasks.named('pipInstall') - dependsOn tasks.matching { it.name == 'generate' } - } - return task - } - - def isortPythonCheck = registerPythonFormatTask('isortPythonCheck', 'isort', '--settings-file', '--check-only') - isortPythonCheck.configure { - dependsOn licensePythonCheck - } - def isortPythonFormat = registerPythonFormatTask('isortPythonFormat', 'isort', '--settings-file') - isortPythonFormat.configure { - dependsOn licensePythonFormat - } - - def blackPythonCheck = registerPythonFormatTask('blackPythonCheck', 'black', '--config', '--check') - blackPythonCheck.configure { - dependsOn licensePythonCheck - dependsOn isortPythonCheck - } - def blackPythonFormat = registerPythonFormatTask('blackPythonFormat', 'black', '--config') - blackPythonFormat.configure { - dependsOn licensePythonFormat - dependsOn isortPythonFormat - } } tasks.register('format') { @@ -348,8 +310,6 @@ allprojects { } tasks.named('check').configure { dependsOn tasks.matching { it.name == 'licensePythonCheck' } - dependsOn tasks.matching { it.name == 'isortPythonCheck' } - dependsOn tasks.matching { it.name == 'blackPythonCheck' } } } From a331dafbf82e7de70147eabb9f0bf83d7747f7ad Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:23:03 -0500 Subject: [PATCH 030/141] make check/fix option required, change name to format --- .../airbyte_ci/{check => format}/commands.py | 11 +++++++---- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/{check => format}/commands.py (88%) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py similarity index 88% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 8e87590f124f..2adfe060bddb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -16,15 +16,18 @@ @click.command() -@click.option("--fix", is_flag=True, default=False, help="Fix any formatting issues detected.") -async def check(fix: bool): +@click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") +async def format(fix: bool): """Checks whether the repository is formatted correctly.""" - success = await run_check(fix) + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await run_format(fix) if not success: click.Abort() -async def run_check(fix: bool) -> bool: +async def run_format(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 328110567c62..8c514cdf3300 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -226,7 +226,7 @@ async def get_modified_files_str(ctx: click.Context): help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", - "check": "pipelines.airbyte_ci.check.commands.check", + "format": "pipelines.airbyte_ci.format.commands.format", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", }, From e9b029f0c8a1d3b267431b45b6c5b6313c45b3a5 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:30:30 -0500 Subject: [PATCH 031/141] format group --- .../pipelines/airbyte_ci/format/commands.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 2adfe060bddb..2b63498b3711 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -15,10 +15,17 @@ from pipelines.helpers.utils import sh_dash_c -@click.command() + +@click.group(help="Commands related to formatting.") @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") -async def format(fix: bool): - """Checks whether the repository is formatted correctly.""" +@click.pass_context +def format(ctx: click.Context, fix: bool): + pass + + +@format.command() +async def python(fix: bool): + """Formats python code via black and isort.""" if fix is None: raise click.UsageError("You must specify either --fix or --check") From 184ca782bf76f891f68d71d921648216f4185a9d Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 19:54:18 -0500 Subject: [PATCH 032/141] pass down context with --fix --- .../pipelines/pipelines/airbyte_ci/format/commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 2b63498b3711..c97f5f96ad61 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -20,12 +20,14 @@ @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click.pass_context def format(ctx: click.Context, fix: bool): - pass + ctx.obj["fix_formatting"] = fix @format.command() -async def python(fix: bool): +@click.pass_context +async def python(ctx: click.Context): """Formats python code via black and isort.""" + fix = ctx.obj.get("fix_formatting") if fix is None: raise click.UsageError("You must specify either --fix or --check") From fc4033989d3640b0d93e0ba0ffde908ab676e44e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 20:36:11 -0500 Subject: [PATCH 033/141] temp-ish? comment out some build.gradle stuff that stops it from working --- .../bases/base-normalization/build.gradle | 46 +++++++++---------- build.gradle | 28 +++++------ octavia-cli/build.gradle | 6 +-- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 5b4d5de4355b..0c733d7cda16 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -38,26 +38,26 @@ generate.configure { tasks.named('check').configure { dependsOn generate } - -[ - 'bigquery', - 'mysql', - 'postgres', - 'redshift', - 'snowflake', - 'oracle', - 'mssql', - 'clickhouse', - 'tidb', - 'duckdb', -].each {destinationName -> - def integrationTestPython = tasks.named('integrationTestPython') - integrationTestPython.configure { - dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') - } - // Not really sure what this task does differently from customIntegrationTestPython, - // but it seems to also run integration tests and as such it depends on the docker images. - integrationTestPython.configure { - dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') - } -} +// +//[ +// 'bigquery', +// 'mysql', +// 'postgres', +// 'redshift', +// 'snowflake', +// 'oracle', +// 'mssql', +// 'clickhouse', +// 'tidb', +// 'duckdb', +//].each {destinationName -> +// def integrationTestPython = tasks.named('integrationTestPython') +// integrationTestPython.configure { +// dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') +// } +// // Not really sure what this task does differently from customIntegrationTestPython, +// // but it seems to also run integration tests and as such it depends on the docker images. +// integrationTestPython.configure { +// dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') +// } +//} diff --git a/build.gradle b/build.gradle index c85b7f324c66..63603f10d3d1 100644 --- a/build.gradle +++ b/build.gradle @@ -230,7 +230,7 @@ allprojects { target javaTarget importOrder() eclipse('4.21').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml')) - licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) +// licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) removeUnusedImports() trimTrailingWhitespace() } @@ -248,19 +248,19 @@ allprojects { dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) } } - def stylingTarget = createFormatTarget(['**/*.yaml', '**/*.yml', '**/*.json']) - if (!stylingTarget.isEmpty()) { - format 'styling', { - target stylingTarget - def npmPath = "${rootProject.tasks.named('npmSetup').get().npmDir.get()}/bin/npm" - def nodePath = "${rootProject.tasks.named('nodeSetup').get().nodeDir.get()}/bin/node" - prettier().npmExecutable(npmPath).nodeExecutable(nodePath) - } - } - } - tasks.matching { it.name =~ /spotless.*/ }.configureEach { - dependsOn tasks.matching { it.name == 'generate' } - } +// def stylingTarget = createFormatTarget(['**/*.yaml', '**/*.yml', '**/*.json']) +// if (!stylingTarget.isEmpty()) { +// format 'styling', { +// target stylingTarget +// def npmPath = "${rootProject.tasks.named('npmSetup').get().npmDir.get()}/bin/npm" +// def nodePath = "${rootProject.tasks.named('nodeSetup').get().nodeDir.get()}/bin/node" +// prettier().npmExecutable(npmPath).nodeExecutable(nodePath) +// } +// } + } +// tasks.matching { it.name =~ /spotless.*/ }.configureEach { +// dependsOn tasks.matching { it.name == 'generate' } +// } tasks.matching { it.name =~ /spotlessStyling.*/ }.configureEach { dependsOn rootProject.tasks.named('nodeSetup') dependsOn rootProject.tasks.named('npmSetup') diff --git a/octavia-cli/build.gradle b/octavia-cli/build.gradle index 0f332639c632..4fb075a628b1 100644 --- a/octavia-cli/build.gradle +++ b/octavia-cli/build.gradle @@ -17,6 +17,6 @@ tasks.register('generate').configure { dependsOn generateApiClient } -tasks.named('installReqs').configure { - dependsOn generateApiClient -} +//tasks.named('installReqs').configure { +// dependsOn generateApiClient +//} From c99bee311db9c291da5adc4323f8a9b5c9b1553f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 24 Oct 2023 20:36:31 -0500 Subject: [PATCH 034/141] add java formatting (no prettier or license format) --- .../pipelines/airbyte_ci/format/commands.py | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index c97f5f96ad61..7e09173ecaa9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -26,17 +26,29 @@ def format(ctx: click.Context, fix: bool): @format.command() @click.pass_context async def python(ctx: click.Context): - """Formats python code via black and isort.""" + """Format python code via black and isort.""" fix = ctx.obj.get("fix_formatting") if fix is None: raise click.UsageError("You must specify either --fix or --check") - success = await run_format(fix) + success = await format_python(fix) if not success: click.Abort() +@format.command() +@click.pass_context +async def java(ctx: click.Context): + """Format java, groovy, and sql code via spotless.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") -async def run_format(fix: bool) -> bool: + success = await format_java(fix) + if not success: + click.Abort() + + +async def format_python(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -89,3 +101,47 @@ async def run_format(fix: bool) -> bool: logger.error("Format failed") logger.error(e.stderr) sys.exit(1) + + +async def format_java(fix: bool) -> bool: + logger = logging.getLogger("format") + if fix: + gradle_command = ["./gradlew", "spotlessApply"] + else: + gradle_command = ["./gradlew", "spotlessCheck", "--scan"] + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + format_container = await ( + dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", "gradle", "**/deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] + ) + ) + .with_workdir(f"/src") + .with_exec(["ls", "-la"]) + .with_exec(gradle_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + + return True + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + return False \ No newline at end of file From 9612d56f223b4e81b2e2c95c599c0556bfd244c3 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 08:38:52 -0500 Subject: [PATCH 035/141] separate commands into subfolders --- .../pipelines/airbyte_ci/format/commands.py | 141 ++---------------- .../airbyte_ci/format/java/commands.py | 66 ++++++++ .../airbyte_ci/format/python/commands.py | 73 +++++++++ 3 files changed, 148 insertions(+), 132 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 7e09173ecaa9..4f94147a362a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -6,142 +6,19 @@ Module exposing the tests command to test airbyte-ci projects. """ -import logging -import sys - import asyncclick as click -import dagger -from pipelines.consts import DOCKER_VERSION -from pipelines.helpers.utils import sh_dash_c - +from pipelines.cli.lazy_group import LazyGroup -@click.group(help="Commands related to formatting.") +@click.group( + cls=LazyGroup, + help="Commands related to formatting.", + lazy_subcommands={ + "python": "pipelines.airbyte_ci.format.python.commands.python", + "java": "pipelines.airbyte_ci.format.java.commands.java", + }, +) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click.pass_context def format(ctx: click.Context, fix: bool): ctx.obj["fix_formatting"] = fix - - -@format.command() -@click.pass_context -async def python(ctx: click.Context): - """Format python code via black and isort.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - - success = await format_python(fix) - if not success: - click.Abort() - -@format.command() -@click.pass_context -async def java(ctx: click.Context): - """Format java, groovy, and sql code via spotless.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - - success = await format_java(fix) - if not success: - click.Abort() - - -async def format_python(fix: bool) -> bool: - """Checks whether the repository is formatted correctly. - Args: - fix (bool): Whether to automatically fix any formatting issues detected. - Returns: - bool: True if the check/format succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] - black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] - if fix: - isort_command.remove("--check-only") - black_command.remove("--check") - - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - format_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.py", "pyproject.toml", "poetry.lock"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"], - ), - ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(isort_command) - .with_exec(black_command) - ) - - await format_container - if fix: - await format_container.directory("/src").export(".") - return True - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - sys.exit(1) - - -async def format_java(fix: bool) -> bool: - logger = logging.getLogger("format") - if fix: - gradle_command = ["./gradlew", "spotlessApply"] - else: - gradle_command = ["./gradlew", "spotlessCheck", "--scan"] - - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - format_container = await ( - dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", "gradle", "**/deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] - ) - ) - .with_workdir(f"/src") - .with_exec(["ls", "-la"]) - .with_exec(gradle_command) - ) - - await format_container - if fix: - await format_container.directory("/src").export(".") - - return True - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - return False \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py new file mode 100644 index 000000000000..092f063f975c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -0,0 +1,66 @@ + +import logging +import sys +import asyncclick as click +import dagger + +from pipelines.helpers.utils import sh_dash_c + + +@click.command() +@click.pass_context +async def java(ctx: click.Context): + """Format java, groovy, and sql code via spotless.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await format_java(fix) + if not success: + click.Abort() + + + + +async def format_java(fix: bool) -> bool: + logger = logging.getLogger("format") + if fix: + gradle_command = ["./gradlew", "spotlessApply"] + else: + gradle_command = ["./gradlew", "spotlessCheck", "--scan"] + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + format_container = await ( + dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", "gradle", "**/deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] + ) + ) + .with_workdir(f"/src") + .with_exec(["ls", "-la"]) + .with_exec(gradle_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + + return True + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + return False \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py new file mode 100644 index 000000000000..8f611d4409f0 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -0,0 +1,73 @@ +import logging +import sys +import asyncclick as click +import dagger + +from pipelines.helpers.utils import sh_dash_c + + +@click.command() +@click.pass_context +async def python(ctx: click.Context): + """Format python code via black and isort.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await format_python(fix) + if not success: + click.Abort() + +async def format_python(fix: bool) -> bool: + """Checks whether the repository is formatted correctly. + Args: + fix (bool): Whether to automatically fix any formatting issues detected. + Returns: + bool: True if the check/format succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] + black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] + if fix: + isort_command.remove("--check-only") + black_command.remove("--check") + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + format_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.py", "pyproject.toml", "poetry.lock"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"], + ), + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(isort_command) + .with_exec(black_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + sys.exit(1) From c93c3dbd9495077787534eb3fc6ba6448ca4eae4 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 08:51:47 -0500 Subject: [PATCH 036/141] remove prettier from gradle file and add command to run prettier --- .../pipelines/airbyte_ci/format/commands.py | 1 + .../airbyte_ci/format/js/commands.py | 73 +++++++++++++++++++ build.gradle | 23 +----- 3 files changed, 75 insertions(+), 22 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 4f94147a362a..270e2914d7d6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -16,6 +16,7 @@ lazy_subcommands={ "python": "pipelines.airbyte_ci.format.python.commands.python", "java": "pipelines.airbyte_ci.format.java.commands.java", + "js": "pipelines.airbyte_ci.format.js.commands.js", }, ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py new file mode 100644 index 000000000000..27399c247542 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -0,0 +1,73 @@ +import logging +import sys +import asyncclick as click +import dagger + +from pipelines.helpers.utils import sh_dash_c + + +@click.command() +@click.pass_context +async def js(ctx: click.Context): + """Format yaml and json code via prettier.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await format_js(fix) + if not success: + click.Abort() + +async def format_js(fix: bool) -> bool: + """Checks whether the repository is formatted correctly. + Args: + fix (bool): Whether to automatically fix any formatting issues detected. + Returns: + bool: True if the check/format succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + + if fix: + prettier_command = ["prettier", "--write", "."] + else: + prettier_command = ["prettier", "--check", "."] + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + format_container = await ( + dagger_client.container() + .from_("node:18.18.0") # Use specified Node.js version + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git", "node_modules"] + ), + ) + .with_workdir(f"/src") + .with_exec(["npm", "install", "-g", "npm@10.1.0"]) + .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) + .with_exec(prettier_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + except Exception as e: + logger.error(f"Failed to format code: {e}") + return False + + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + sys.exit(1) diff --git a/build.gradle b/build.gradle index 63603f10d3d1..c38c55fc7d51 100644 --- a/build.gradle +++ b/build.gradle @@ -104,19 +104,6 @@ def createJavaLicenseWith = { license -> return createLicenseWith(license, '/*', ' */', ' * ', false) } -// node is required by the root project to apply the prettier formatter repo-wide. -node { - download = true - version = '18.18.0' - npmVersion = '10.1.0' - // When setting both these directories, npm and node will be in separate directories. - workDir = file("${buildDir}/nodejs") - npmWorkDir = file("${buildDir}/npm") - // Do not declare the repository. - distBaseUrl = null - -} - // python is required by the root project to apply python formatters like isort or black. python { envPath = '.venv' @@ -248,15 +235,7 @@ allprojects { dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) } } -// def stylingTarget = createFormatTarget(['**/*.yaml', '**/*.yml', '**/*.json']) -// if (!stylingTarget.isEmpty()) { -// format 'styling', { -// target stylingTarget -// def npmPath = "${rootProject.tasks.named('npmSetup').get().npmDir.get()}/bin/npm" -// def nodePath = "${rootProject.tasks.named('nodeSetup').get().nodeDir.get()}/bin/node" -// prettier().npmExecutable(npmPath).nodeExecutable(nodePath) -// } -// } + // TODO: figure out why prettier has so many problems with files now that it didn't before } // tasks.matching { it.name =~ /spotless.*/ }.configureEach { // dependsOn tasks.matching { it.name == 'generate' } From e7d4cead629359180133a504717d06901bfb4ff7 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 09:34:35 -0500 Subject: [PATCH 037/141] add default ignore list which fixes prettier. for some reason i can't remove it from gradle or java breaks --- .../pipelines/airbyte_ci/format/__init__.py | 0 .../pipelines/airbyte_ci/format/consts.py | 43 +++++++++++++++++++ .../airbyte_ci/format/java/commands.py | 3 +- .../airbyte_ci/format/js/commands.py | 3 +- .../airbyte_ci/format/python/commands.py | 3 +- build.gradle | 1 - 6 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py new file mode 100644 index 000000000000..6a23a57f77fc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +DEFAULT_FORMAT_IGNORE_LIST = [ + '**/__pycache__', + '"**/.pytest_cache', + "**/.venv", + "**/venv", + '**/.gradle', + '**/node_modules', + '**/.eggs', + '**/.mypy_cache', + '**/.venv', + '**/*.egg-info', + '**/build', + '**/dbt-project-template', + '**/dbt-project-template-mssql', + '**/dbt-project-template-mysql', + '**/dbt-project-template-oracle', + '**/dbt-project-template-clickhouse', + '**/dbt-project-template-snowflake', + '**/dbt-project-template-tidb', + '**/dbt-project-template-duckdb', + '**/dbt_test_config', + '**/normalization_test_output', + # '**/tools', + '**/secrets', + '**/charts', # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + '**/resources/seed/*_catalog.json', # Do not remove - this is also necessary to prevent diffs in our github workflows + '**/resources/seed/*_registry.json', # Do not remove - this is also necessary to prevent diffs in our github workflows + '**/resources/seed/specs_secrets_mask.yaml', # Downloaded externally. + '**/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py', + '**/source-amplitude/unit_tests/api_data/zipped.json', # Zipped file presents as non-UTF-8 making spotless sad + '**/airbyte-connector-builder-server/connector_builder/generated', # autogenerated code doesn't need to be formatted + '**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', # These are deliberately invalid and unformattable. + '**/__init__.py', + '**/declarative_component_schema.py', + '**/source-stock-ticker-api-tutorial/source.py', + '**/tools/git_hooks/tests/test_spec_linter.py', + '**/tools/schema_generator/schema_generator/infer_schemas.py', + "**/.git", + ] \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 092f063f975c..23e1e95aeaee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -3,6 +3,7 @@ import sys import asyncclick as click import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.helpers.utils import sh_dash_c @@ -47,7 +48,7 @@ async def format_java(fix: bool) -> bool: dagger_client.host().directory( ".", include=["**/*.java", "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", "gradle", "**/deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"] + exclude=DEFAULT_FORMAT_IGNORE_LIST ) ) .with_workdir(f"/src") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index 27399c247542..37f66f624691 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -4,6 +4,7 @@ import dagger from pipelines.helpers.utils import sh_dash_c +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST @click.command() @@ -50,7 +51,7 @@ async def format_js(fix: bool) -> bool: dagger_client.host().directory( ".", include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git", "node_modules"] + exclude=DEFAULT_FORMAT_IGNORE_LIST ), ) .with_workdir(f"/src") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index 8f611d4409f0..6b190f2bb531 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -2,6 +2,7 @@ import sys import asyncclick as click import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.helpers.utils import sh_dash_c @@ -54,7 +55,7 @@ async def format_python(fix: bool) -> bool: dagger_client.host().directory( ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**/build", ".git"], + exclude=DEFAULT_FORMAT_IGNORE_LIST ), ) .with_workdir(f"/src") diff --git a/build.gradle b/build.gradle index c38c55fc7d51..2b89d1a185e2 100644 --- a/build.gradle +++ b/build.gradle @@ -235,7 +235,6 @@ allprojects { dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) } } - // TODO: figure out why prettier has so many problems with files now that it didn't before } // tasks.matching { it.name =~ /spotless.*/ }.configureEach { // dependsOn tasks.matching { it.name == 'generate' } From 742556345732d5477a90d925d43cef7b036cc669 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 09:49:51 -0500 Subject: [PATCH 038/141] add default ignore list which fixes prettier. for some reason i can't remove it from gradle or java breaks --- .../pipelines/airbyte_ci/format/commands.py | 2 +- .../pipelines/airbyte_ci/format/consts.py | 76 +++++++++---------- .../airbyte_ci/format/java/commands.py | 27 ++++--- .../airbyte_ci/format/js/commands.py | 7 +- .../airbyte_ci/format/python/commands.py | 7 +- build.gradle | 49 ------------ 6 files changed, 64 insertions(+), 104 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 270e2914d7d6..bbcaa4fff282 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -7,9 +7,9 @@ """ import asyncclick as click - from pipelines.cli.lazy_group import LazyGroup + @click.group( cls=LazyGroup, help="Commands related to formatting.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py index 6a23a57f77fc..e0092fef8357 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -3,41 +3,41 @@ # DEFAULT_FORMAT_IGNORE_LIST = [ - '**/__pycache__', - '"**/.pytest_cache', - "**/.venv", - "**/venv", - '**/.gradle', - '**/node_modules', - '**/.eggs', - '**/.mypy_cache', - '**/.venv', - '**/*.egg-info', - '**/build', - '**/dbt-project-template', - '**/dbt-project-template-mssql', - '**/dbt-project-template-mysql', - '**/dbt-project-template-oracle', - '**/dbt-project-template-clickhouse', - '**/dbt-project-template-snowflake', - '**/dbt-project-template-tidb', - '**/dbt-project-template-duckdb', - '**/dbt_test_config', - '**/normalization_test_output', - # '**/tools', - '**/secrets', - '**/charts', # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - '**/resources/seed/*_catalog.json', # Do not remove - this is also necessary to prevent diffs in our github workflows - '**/resources/seed/*_registry.json', # Do not remove - this is also necessary to prevent diffs in our github workflows - '**/resources/seed/specs_secrets_mask.yaml', # Downloaded externally. - '**/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py', - '**/source-amplitude/unit_tests/api_data/zipped.json', # Zipped file presents as non-UTF-8 making spotless sad - '**/airbyte-connector-builder-server/connector_builder/generated', # autogenerated code doesn't need to be formatted - '**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', # These are deliberately invalid and unformattable. - '**/__init__.py', - '**/declarative_component_schema.py', - '**/source-stock-ticker-api-tutorial/source.py', - '**/tools/git_hooks/tests/test_spec_linter.py', - '**/tools/schema_generator/schema_generator/infer_schemas.py', - "**/.git", - ] \ No newline at end of file + "**/__pycache__", + '"**/.pytest_cache', + "**/.venv", + "**/venv", + "**/.gradle", + "**/node_modules", + "**/.eggs", + "**/.mypy_cache", + "**/.venv", + "**/*.egg-info", + "**/build", + "**/dbt-project-template", + "**/dbt-project-template-mssql", + "**/dbt-project-template-mysql", + "**/dbt-project-template-oracle", + "**/dbt-project-template-clickhouse", + "**/dbt-project-template-snowflake", + "**/dbt-project-template-tidb", + "**/dbt-project-template-duckdb", + "**/dbt_test_config", + "**/normalization_test_output", + # '**/tools', + "**/secrets", + "**/charts", # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + "**/resources/seed/*_catalog.json", # Do not remove - this is also necessary to prevent diffs in our github workflows + "**/resources/seed/*_registry.json", # Do not remove - this is also necessary to prevent diffs in our github workflows + "**/resources/seed/specs_secrets_mask.yaml", # Downloaded externally. + "**/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py", + "**/source-amplitude/unit_tests/api_data/zipped.json", # Zipped file presents as non-UTF-8 making spotless sad + "**/airbyte-connector-builder-server/connector_builder/generated", # autogenerated code doesn't need to be formatted + "**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # These are deliberately invalid and unformattable. + "**/__init__.py", + "**/declarative_component_schema.py", + "**/source-stock-ticker-api-tutorial/source.py", + "**/tools/git_hooks/tests/test_spec_linter.py", + "**/tools/schema_generator/schema_generator/infer_schemas.py", + "**/.git", +] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 23e1e95aeaee..1c2dc77e794c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -1,10 +1,9 @@ - import logging import sys + import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST - from pipelines.helpers.utils import sh_dash_c @@ -21,8 +20,6 @@ async def java(ctx: click.Context): click.Abort() - - async def format_java(fix: bool) -> bool: logger = logging.getLogger("format") if fix: @@ -46,10 +43,22 @@ async def format_java(fix: bool) -> bool: .with_mounted_directory( "/src", dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", "gradle", "**/deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties"], - exclude=DEFAULT_FORMAT_IGNORE_LIST - ) + ".", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), ) .with_workdir(f"/src") .with_exec(["ls", "-la"]) @@ -64,4 +73,4 @@ async def format_java(fix: bool) -> bool: except dagger.ExecError as e: logger.error("Format failed") logger.error(e.stderr) - return False \ No newline at end of file + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index 37f66f624691..f41099513774 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -1,10 +1,10 @@ import logging import sys + import asyncclick as click import dagger - -from pipelines.helpers.utils import sh_dash_c from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.helpers.utils import sh_dash_c @click.command() @@ -19,6 +19,7 @@ async def js(ctx: click.Context): if not success: click.Abort() + async def format_js(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: @@ -51,7 +52,7 @@ async def format_js(fix: bool) -> bool: dagger_client.host().directory( ".", include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST + exclude=DEFAULT_FORMAT_IGNORE_LIST, ), ) .with_workdir(f"/src") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index 6b190f2bb531..a496baef825c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -1,9 +1,9 @@ import logging import sys + import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST - from pipelines.helpers.utils import sh_dash_c @@ -19,6 +19,7 @@ async def python(ctx: click.Context): if not success: click.Abort() + async def format_python(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: @@ -53,9 +54,7 @@ async def format_python(fix: bool) -> bool: .with_mounted_directory( "/src", dagger_client.host().directory( - ".", - include=["**/*.py", "pyproject.toml", "poetry.lock"], - exclude=DEFAULT_FORMAT_IGNORE_LIST + ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST ), ) .with_workdir(f"/src") diff --git a/build.gradle b/build.gradle index 2b89d1a185e2..3e78181b9493 100644 --- a/build.gradle +++ b/build.gradle @@ -104,52 +104,6 @@ def createJavaLicenseWith = { license -> return createLicenseWith(license, '/*', ' */', ' * ', false) } -// python is required by the root project to apply python formatters like isort or black. -python { - envPath = '.venv' - minPythonVersion = '3.10' // should be 3.10 for local development - - // Amazon Linux support. - // The airbyte-ci tool runs gradle tasks in AL2023-based containers. - // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. - // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. - try { - if ("python3.11 --version".execute().waitFor() == 0) { - // python3.11 definitely exists at this point, use it instead of 'python3'. - pythonBinary "python3.11" - } - } catch (IOException _) { - // Swallow exception if python3.11 is not installed. - } - // Pyenv support. - try { - def pyenvRoot = "pyenv root".execute() - def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() - // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. - pythonBinary "python" - if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { - pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" - } - } catch (IOException _) { - // Swallow exception if pyenv is not installed. - } - - scope = 'VIRTUALENV' - installVirtualenv = true - // black and isort are required for formatting - pip 'black:22.3.0' - pip 'isort:5.6.4' - // poetry is required for installing and running airbyte-ci - pip 'poetry:1.5.1' -} -def cleanPythonVenv = rootProject.tasks.register('cleanPythonVenv', Exec) { - commandLine 'rm' - args '-rf', "${rootProject.projectDir.absolutePath}/.venv" -} -rootProject.tasks.named('clean').configure { - dependsOn cleanPythonVenv -} - // format tasks per project allprojects { @@ -585,9 +539,6 @@ def poetryCleanVirtualenv = tasks.register('cleanVirtualenv', Exec) { rootProject.file('.venv/bin/python').exists() } } -cleanPythonVenv.configure { - dependsOn poetryCleanVirtualenv -} subprojects { if (!isConnectorProject(project)) { From 5e0852653ffde80101d28ca2aebd4298381911f8 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 09:52:00 -0500 Subject: [PATCH 039/141] change clean command --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index 3e78181b9493..bd097306bff9 100644 --- a/build.gradle +++ b/build.gradle @@ -539,6 +539,9 @@ def poetryCleanVirtualenv = tasks.register('cleanVirtualenv', Exec) { rootProject.file('.venv/bin/python').exists() } } +clean.configure { + dependsOn poetryCleanVirtualenv +} subprojects { if (!isConnectorProject(project)) { From 887f7ce3250891d5aceb46116a7fa83b03c66940 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 10:51:48 -0500 Subject: [PATCH 040/141] temp: add license check --- .../pipelines/airbyte_ci/format/commands.py | 1 + .../pipelines/airbyte_ci/format/consts.py | 1 + .../airbyte_ci/format/python/commands.py | 61 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index bbcaa4fff282..7015ce092fc3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -17,6 +17,7 @@ "python": "pipelines.airbyte_ci.format.python.commands.python", "java": "pipelines.airbyte_ci.format.java.commands.java", "js": "pipelines.airbyte_ci.format.js.commands.js", + "license": "pipelines.airbyte_ci.format.python.commands.license", }, ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py index e0092fef8357..add25c815518 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -9,6 +9,7 @@ "**/venv", "**/.gradle", "**/node_modules", + "**/.tox", "**/.eggs", "**/.mypy_cache", "**/.venv", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index a496baef825c..c7e175135527 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -20,6 +20,18 @@ async def python(ctx: click.Context): click.Abort() +@click.command() +@click.pass_context +async def license(ctx: click.Context): + """Add license to python and java code via addlicense.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await format_license(fix) + if not success: + click.Abort() + async def format_python(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: @@ -71,3 +83,52 @@ async def format_python(fix: bool) -> bool: logger.error("Format failed") logger.error(e.stderr) sys.exit(1) + + + + +async def format_license(fix: bool = False) -> bool: + license_text = "LICENSE_SHORT" + logger = logging.getLogger(f"format") + + + if fix: + addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "." "*.java", "*.py"] + else: + addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} -check ."] + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + license_container = await ( + dagger_client.container() + .from_("golang:1.17") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash tree", + "go get -u github.com/google/addlicense" + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["ls", "-la"]) + .with_exec(["tree", "."]) + .with_exec(addlicense_command) + ) + + await license_container + if fix: + await license_container.directory("/src").export(".") + return True + except Exception as e: + logger.error(f"Failed to apply license: {e}") + return False \ No newline at end of file From e815cd6cbac4eec0935492ae1f992364fba6358a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 10:55:49 -0500 Subject: [PATCH 041/141] fix fix command --- .../pipelines/pipelines/airbyte_ci/format/python/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index c7e175135527..36bad876afe7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -93,7 +93,7 @@ async def format_license(fix: bool = False) -> bool: if fix: - addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "." "*.java", "*.py"] + addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} ."] else: addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} -check ."] From e05ad0a61412949d5aa378d70e748f6d47773f0a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 10:57:41 -0500 Subject: [PATCH 042/141] modify short license for python --- LICENSE_SHORT | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LICENSE_SHORT b/LICENSE_SHORT index dc7ac8476682..144d8e44fa34 100644 --- a/LICENSE_SHORT +++ b/LICENSE_SHORT @@ -1 +1,3 @@ + Copyright (c) 2023 Airbyte, Inc., all rights reserved. + From 285cefd1f91736fefadd8a3e504da296c31d7392 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 11:09:07 -0500 Subject: [PATCH 043/141] modify short license again? --- LICENSE_SHORT | 2 -- 1 file changed, 2 deletions(-) diff --git a/LICENSE_SHORT b/LICENSE_SHORT index 144d8e44fa34..dc7ac8476682 100644 --- a/LICENSE_SHORT +++ b/LICENSE_SHORT @@ -1,3 +1 @@ - Copyright (c) 2023 Airbyte, Inc., all rights reserved. - From 86d358b05fb6db00b6835c11923a5a96f0a40c45 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 11:13:35 -0500 Subject: [PATCH 044/141] move command to own file --- .../pipelines/airbyte_ci/format/commands.py | 2 +- .../airbyte_ci/format/license/commands.py | 68 +++++++++++++++++++ .../airbyte_ci/format/python/commands.py | 61 ----------------- 3 files changed, 69 insertions(+), 62 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 7015ce092fc3..18a58f79d2ad 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -17,7 +17,7 @@ "python": "pipelines.airbyte_ci.format.python.commands.python", "java": "pipelines.airbyte_ci.format.java.commands.java", "js": "pipelines.airbyte_ci.format.js.commands.js", - "license": "pipelines.airbyte_ci.format.python.commands.license", + "license": "pipelines.airbyte_ci.format.license.commands.license", }, ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py new file mode 100644 index 000000000000..4ccabcaf182f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import sys +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST + +from pipelines.helpers.utils import sh_dash_c + + +@click.command() +@click.pass_context +async def license(ctx: click.Context): + """Add license to python and java code via addlicense.""" + fix = ctx.obj.get("fix_formatting") + if fix is None: + raise click.UsageError("You must specify either --fix or --check") + + success = await format_license(fix) + if not success: + click.Abort() + +async def format_license(fix: bool = False) -> bool: + license_text = "LICENSE_SHORT" + logger = logging.getLogger(f"format") + + + if fix: + addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} ."] + else: + addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} -check ."] + + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + license_container = await ( + dagger_client.container() + .from_("golang:1.17") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash tree", + "go get -u github.com/google/addlicense" + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["ls", "-la"]) + .with_exec(["tree", "."]) + .with_exec(addlicense_command) + ) + + await license_container + if fix: + await license_container.directory("/src").export(".") + return True + except Exception as e: + logger.error(f"Failed to apply license: {e}") + return False \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index 36bad876afe7..a496baef825c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -20,18 +20,6 @@ async def python(ctx: click.Context): click.Abort() -@click.command() -@click.pass_context -async def license(ctx: click.Context): - """Add license to python and java code via addlicense.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - - success = await format_license(fix) - if not success: - click.Abort() - async def format_python(fix: bool) -> bool: """Checks whether the repository is formatted correctly. Args: @@ -83,52 +71,3 @@ async def format_python(fix: bool) -> bool: logger.error("Format failed") logger.error(e.stderr) sys.exit(1) - - - - -async def format_license(fix: bool = False) -> bool: - license_text = "LICENSE_SHORT" - logger = logging.getLogger(f"format") - - - if fix: - addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} ."] - else: - addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} -check ."] - - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - license_container = await ( - dagger_client.container() - .from_("golang:1.17") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash tree", - "go get -u github.com/google/addlicense" - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(["ls", "-la"]) - .with_exec(["tree", "."]) - .with_exec(addlicense_command) - ) - - await license_container - if fix: - await license_container.directory("/src").export(".") - return True - except Exception as e: - logger.error(f"Failed to apply license: {e}") - return False \ No newline at end of file From b04fa719c309efefc2cf3217575def11dafaef09 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 11:16:12 -0500 Subject: [PATCH 045/141] remove bash -c and ls/tree --- .../pipelines/pipelines/airbyte_ci/format/java/commands.py | 1 - .../pipelines/airbyte_ci/format/license/commands.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 1c2dc77e794c..91dc3273a2de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -61,7 +61,6 @@ async def format_java(fix: bool) -> bool: ), ) .with_workdir(f"/src") - .with_exec(["ls", "-la"]) .with_exec(gradle_command) ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py index 4ccabcaf182f..ad7e1cd90e47 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -27,9 +27,9 @@ async def format_license(fix: bool = False) -> bool: if fix: - addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} ."] + addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] else: - addlicense_command = ["bash", "-c", f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_text} -check ."] + addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: try: @@ -54,8 +54,6 @@ async def format_license(fix: bool = False) -> bool: ), ) .with_workdir(f"/src") - .with_exec(["ls", "-la"]) - .with_exec(["tree", "."]) .with_exec(addlicense_command) ) From 116442cbf4be63a209aa78b0e0d3e5c7cfd3422e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 11:19:56 -0500 Subject: [PATCH 046/141] run python format --- .../airbyte_ci/format/license/commands.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py index ad7e1cd90e47..eae3b56ea783 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -2,10 +2,10 @@ import logging import sys + import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST - from pipelines.helpers.utils import sh_dash_c @@ -21,11 +21,11 @@ async def license(ctx: click.Context): if not success: click.Abort() + async def format_license(fix: bool = False) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") - if fix: addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] else: @@ -36,15 +36,7 @@ async def format_license(fix: bool = False) -> bool: license_container = await ( dagger_client.container() .from_("golang:1.17") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash tree", - "go get -u github.com/google/addlicense" - ] - ) - ) + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) .with_mounted_directory( "/src", dagger_client.host().directory( @@ -63,4 +55,4 @@ async def format_license(fix: bool = False) -> bool: return True except Exception as e: logger.error(f"Failed to apply license: {e}") - return False \ No newline at end of file + return False From 9ead19c1129cd79e98680e7f023afc0b8c735614 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 11:25:38 -0500 Subject: [PATCH 047/141] remove license stuff from gradle build file --- build.gradle | 78 ---------------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/build.gradle b/build.gradle index bd097306bff9..1e772365e123 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,4 @@ import com.github.spotbugs.snom.SpotBugsTask -import ru.vyarus.gradle.plugin.python.task.PythonTask -import com.hierynomus.gradle.license.tasks.LicenseCheck -import com.hierynomus.gradle.license.tasks.LicenseFormat // The buildscript block defines dependencies in order for .gradle file evaluation. // This is separate from application dependencies. @@ -9,7 +6,6 @@ import com.hierynomus.gradle.license.tasks.LicenseFormat buildscript { dependencies { classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.20.0' - classpath 'gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.16.1' } } @@ -67,43 +63,6 @@ allprojects { version = rootProject.ext.version } -// License generation logic. -def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython -> - /* - In java, we don't have a second linter/styling tool other than spotless so it doesn't really - matter if we write a newline or not for startComment/endComment. - - However, in python, we are using black that double-checks and reformats the code. - Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a - big deal and would be reformatted (removed) because of black's specs. - */ - def tmp = File.createTempFile('tmp', '.tmp') - tmp.withWriter { - def w = it - if (startComment.length() > 0 || !isPython) { - w.writeLine(startComment) - } - license.eachLine { - w << lineComment - w.writeLine(it) - } - if (endComment.length() > 0 || !isPython) { - w.writeLine(endComment) - } - w.writeLine("") - if (isPython) { - w.writeLine("") - } - } - return tmp -} -def createPythonLicenseWith = { license -> - return createLicenseWith(license, '', '', '', true) -} -def createJavaLicenseWith = { license -> - return createLicenseWith(license, '/*', ' */', ' * ', false) -} - // format tasks per project allprojects { @@ -171,7 +130,6 @@ allprojects { target javaTarget importOrder() eclipse('4.21').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml')) -// licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) removeUnusedImports() trimTrailingWhitespace() } @@ -202,47 +160,11 @@ allprojects { enabled = false } - // python license header generation is part of 'format' - def pythonTarget = createFormatTarget('**/*.py') - if (!project.name.endsWith('source-stock-ticker-api-tutorial') && !pythonTarget.isEmpty()) { - - apply plugin: 'com.github.hierynomus.license' - - license { - header rootProject.file("LICENSE_SHORT") - } - - // Disable auto-generated java tasks, we rely on spotless there. - tasks.matching { it.name =~ /license.*/ }.configureEach { - enabled = false - } - - def licensePythonCheck = tasks.register('licensePythonCheck', LicenseCheck) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = pythonTarget - strictCheck = true - } - licensePythonCheck.configure { - dependsOn tasks.matching { it.name == 'generate' } - } - def licensePythonFormat = tasks.register('licensePythonFormat', LicenseFormat) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = pythonTarget - strictCheck = true - } - licensePythonFormat.configure { - dependsOn tasks.matching { it.name == 'generate' } - } - } - tasks.register('format') { dependsOn tasks.matching { it.name == 'generate' } dependsOn tasks.matching { it.name =~ /spotless.*Apply/ } dependsOn tasks.matching { it.name =~ /.*PythonFormat/ } } - tasks.named('check').configure { - dependsOn tasks.matching { it.name == 'licensePythonCheck' } - } } def getCDKTargetVersion() { From 44c506496656f14bc352e7c3c6620fb89aca502f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 12:34:13 -0500 Subject: [PATCH 048/141] clean up gradle ignores for format --- build.gradle | 45 ++------------------------------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/build.gradle b/build.gradle index 1e772365e123..ba35032d1fd2 100644 --- a/build.gradle +++ b/build.gradle @@ -67,49 +67,8 @@ allprojects { allprojects { def createFormatTarget = { pattern -> - // We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a - // monorepo setup and it doesn't actually exclude directories reliably. - // This code makes the behavior predictable. - ArrayList excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt-project-template-clickhouse', - 'dbt-project-template-snowflake', - 'dbt-project-template-tidb', - 'dbt-project-template-duckdb', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'resources/seed/*_registry.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'resources/seed/specs_secrets_mask.yaml', // Downloaded externally. - 'resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py', - 'source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad - 'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted - 'airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', // These are deliberately invalid and unformattable. - '__init__.py', - 'declarative_component_schema.py', - 'source-stock-ticker-api-tutorial/source.py', - 'tools/git_hooks/tests/test_spec_linter.py', - 'tools/schema_generator/schema_generator/infer_schemas.py', - ] - // Ensure that the excludes work when we're already partly in their path. - excludes.addAll excludes.collectMany { - it.startsWith("${project.name}/") ? [it, it.substring("${project.name}/".length())] : [it] - } - // Prefix everything with double-globs because we only care about path suffixes. - excludes = excludes.collect { "**/${it}" } + // Ignore new build files as well as the ones ignored when running this through airbyte-ci + ArrayList excludes = ['**/build'] // Remove whatever's covered by the project's subprojects, so that each file is targeted by at most one task. def currentProject = project project.subprojects { From f929b9db812ed57c85e59446bb9c5e7392991edd Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 12:51:31 -0500 Subject: [PATCH 049/141] simplify gradle file --- build.gradle | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index ba35032d1fd2..cd759be23a5e 100644 --- a/build.gradle +++ b/build.gradle @@ -69,15 +69,8 @@ allprojects { def createFormatTarget = { pattern -> // Ignore new build files as well as the ones ignored when running this through airbyte-ci ArrayList excludes = ['**/build'] - // Remove whatever's covered by the project's subprojects, so that each file is targeted by at most one task. - def currentProject = project - project.subprojects { - if (it.parent.projectDir == currentProject.projectDir) { - excludes.add "${it.projectDir}/**" - } - } // Build the FileTree. - return fileTree(dir: projectDir, include: pattern, exclude: excludes) + return fileTree(dir: projectDir, include: pattern, exclude: ['**/build']) } // Apply spotless formatting. From dce7dc1b9df425fa8f43b7b26ac3bf0cef9cba72 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Wed, 25 Oct 2023 15:51:53 -0500 Subject: [PATCH 050/141] allow invoking without subcommand to call all commands --- .../pipelines/airbyte_ci/format/commands.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 18a58f79d2ad..f793ca043724 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -14,13 +14,26 @@ cls=LazyGroup, help="Commands related to formatting.", lazy_subcommands={ - "python": "pipelines.airbyte_ci.format.python.commands.python", "java": "pipelines.airbyte_ci.format.java.commands.java", "js": "pipelines.airbyte_ci.format.js.commands.js", "license": "pipelines.airbyte_ci.format.license.commands.license", + "python": "pipelines.airbyte_ci.format.python.commands.python", }, + invoke_without_command=True, ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click.pass_context -def format(ctx: click.Context, fix: bool): +async def format(ctx: click.Context, fix: bool): + from pipelines.airbyte_ci.format.java.commands import java + from pipelines.airbyte_ci.format.js.commands import js + from pipelines.airbyte_ci.format.license.commands import license + from pipelines.airbyte_ci.format.python.commands import python + ctx.obj["fix_formatting"] = fix + + if ctx.invoked_subcommand is None: + # TODO: ctx.forward should forward the fix commands to the subcommands + await ctx.invoke(license) + await ctx.invoke(java) + await ctx.invoke(js) + await ctx.invoke(python) From 8e874001841e8a5826ba81e5bf467fdc1dc6a223 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 26 Oct 2023 11:52:50 -0500 Subject: [PATCH 051/141] use smaller and supported docker images for python and node --- .../pipelines/pipelines/airbyte_ci/format/js/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/python/commands.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index f41099513774..8e24bcdc56ce 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -38,7 +38,7 @@ async def format_js(fix: bool) -> bool: try: format_container = await ( dagger_client.container() - .from_("node:18.18.0") # Use specified Node.js version + .from_("node:18.18.0-slim") .with_exec( sh_dash_c( [ diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index a496baef825c..2db5d87d353f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -38,7 +38,7 @@ async def format_python(fix: bool) -> bool: try: format_container = await ( dagger_client.container() - .from_("python:3.10.12") + .from_("python:3.10.13-slim") .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") .with_exec( sh_dash_c( From 7ad4f1bdf7ff25a5cf067858e6699e24977a785f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 10:33:01 -0500 Subject: [PATCH 052/141] remove more spotless stuff from gradle --- build.gradle | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/build.gradle b/build.gradle index cd759be23a5e..6d51da57c83e 100644 --- a/build.gradle +++ b/build.gradle @@ -100,23 +100,6 @@ allprojects { } } } -// tasks.matching { it.name =~ /spotless.*/ }.configureEach { -// dependsOn tasks.matching { it.name == 'generate' } -// } - tasks.matching { it.name =~ /spotlessStyling.*/ }.configureEach { - dependsOn rootProject.tasks.named('nodeSetup') - dependsOn rootProject.tasks.named('npmSetup') - } - // TODO: remove this once the CI routinely enforces formatting checks. - tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { - enabled = false - } - - tasks.register('format') { - dependsOn tasks.matching { it.name == 'generate' } - dependsOn tasks.matching { it.name =~ /spotless.*Apply/ } - dependsOn tasks.matching { it.name =~ /.*PythonFormat/ } - } } def getCDKTargetVersion() { From 86cd43f089af3079d717daa2f8fdbfa2dc57004b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 11:18:17 -0500 Subject: [PATCH 053/141] remove format from gradle check workflow --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 35583de7398d..74d11a3c9b0a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -99,7 +99,7 @@ jobs: # TODO: be able to remove the skipSlowTests property # TODO: remove the format task ASAP # TODO: roll the CDK tasks into check - arguments: --scan --no-daemon --no-watch-fs format check -DskipSlowTests=true + arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true # In case of self-hosted EC2 errors, remove this block. stop-check-runner: From 5d78648444a42deb25241dcc203617b39d815d36 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 11:21:21 -0500 Subject: [PATCH 054/141] bring python back to build.gradle file but without formatting deps --- build.gradle | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6d51da57c83e..6ff8c4d2b84e 100644 --- a/build.gradle +++ b/build.gradle @@ -63,6 +63,50 @@ allprojects { version = rootProject.ext.version } +// python is required by the root project to run CAT tests for connectors +python { + envPath = '.venv' + minPythonVersion = '3.10' // should be 3.10 for local development + + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } + // Pyenv support. + try { + def pyenvRoot = "pyenv root".execute() + def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() + // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. + pythonBinary "python" + if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { + pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" + } + } catch (IOException _) { + // Swallow exception if pyenv is not installed. + } + + scope = 'VIRTUALENV' + installVirtualenv = true + // poetry is required for installing and running airbyte-ci + pip 'poetry:1.5.1' +} + +def cleanPythonVenv = rootProject.tasks.register('cleanPythonVenv', Exec) { + commandLine 'rm' + args '-rf', "${rootProject.projectDir.absolutePath}/.venv" +} +rootProject.tasks.named('clean').configure { + dependsOn cleanPythonVenv +} + // format tasks per project allprojects { @@ -396,7 +440,7 @@ def poetryCleanVirtualenv = tasks.register('cleanVirtualenv', Exec) { rootProject.file('.venv/bin/python').exists() } } -clean.configure { +cleanPythonVenv.configure { dependsOn poetryCleanVirtualenv } From 41517263ef1175d392be4879756be1d3d84a80e5 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 11:40:44 -0500 Subject: [PATCH 055/141] test: new github action workflow --- .github/workflows/format.yml | 66 +++++++----------------------------- 1 file changed, 12 insertions(+), 54 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8854146c2b9e..c217c54ca667 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,4 +1,4 @@ -name: Format Code (Python + Java) +name: Check for formatting errors concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -10,9 +10,9 @@ on: - master pull_request: jobs: - format-and-commit: + format-check: runs-on: ubuntu-latest - name: "Apply All Formatting Rules" + name: "Check All Formatting Rules" timeout-minutes: 40 steps: - name: Checkout Airbyte @@ -20,16 +20,16 @@ jobs: with: ref: ${{ github.head_ref }} # Important that this is set so that CI checks are triggered again - # Without this we would be be forever waiting on required checks to pass + # Without this we would be forever waiting on required checks to pass token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - # IMPORTANT! This is nessesary to make sure that a status is reported on the PR - # even if the workflow is skipped. If we used github actions filters, the workflow + # IMPORTANT! This is necessary to make sure that a status is reported on the PR + # even if the workflow is skipped. If we used GitHub Actions filters, the workflow # would not be reported as skipped, but instead would be forever pending. # # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. # - # Also it gets worse + # Also, it gets worse # # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. @@ -42,59 +42,17 @@ jobs: - '**/*' - '!**/*.md' - - uses: actions/setup-java@v3 + - name: Run airbyte-ci formatting checks + uses: ./.github/actions/run-dagger-pipeline with: - distribution: "zulu" - java-version: "17" - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties < Date: Mon, 30 Oct 2023 11:42:07 -0500 Subject: [PATCH 056/141] add dockerhub and sentry auth --- .github/workflows/format.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index c217c54ca667..4de2035f00b3 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -46,6 +46,9 @@ jobs: uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} subcommand: "format --check" notify-failure-slack-channel: From b08db8a14ab1553b3762059897f205caf5874cc5 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 12:51:36 -0500 Subject: [PATCH 057/141] maybe its available on the runner? --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 4de2035f00b3..d976dbea49af 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -11,7 +11,7 @@ on: pull_request: jobs: format-check: - runs-on: ubuntu-latest + runs-on: medium-runner name: "Check All Formatting Rules" timeout-minutes: 40 steps: From de086cd195a5ed05f88f0800414ed7041fe18004 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 13:15:23 -0500 Subject: [PATCH 058/141] use the xl babyyy --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index d976dbea49af..7f3ed1ce6cad 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -11,7 +11,7 @@ on: pull_request: jobs: format-check: - runs-on: medium-runner + runs-on: "conn-prod-xlarge-runner" name: "Check All Formatting Rules" timeout-minutes: 40 steps: From fc5a391c4b1f4b3bb057bc935ff0f1495e3dbfab Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 30 Oct 2023 13:37:18 -0500 Subject: [PATCH 059/141] all the tokens? --- .github/workflows/format.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 7f3ed1ce6cad..42129c4652f2 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -48,7 +48,9 @@ jobs: context: "pull_request" docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} subcommand: "format --check" notify-failure-slack-channel: From e59f646a2a4ebf67808dd9cd52d4531a54968d51 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 13:08:58 -0500 Subject: [PATCH 060/141] fix java command with clickpipelinecontext --- .../pipelines/airbyte_ci/format/commands.py | 4 +- .../airbyte_ci/format/java/commands.py | 102 +++++++++--------- 2 files changed, 56 insertions(+), 50 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index f793ca043724..3aaa6a60df6a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -7,6 +7,7 @@ """ import asyncclick as click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup @@ -22,7 +23,8 @@ invoke_without_command=True, ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") -@click.pass_context +@click_merge_args_into_context_obj +@click_ignore_unused_kwargs async def format(ctx: click.Context, fix: bool): from pipelines.airbyte_ci.format.java.commands import java from pipelines.airbyte_ci.format.js.commands import js diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 91dc3273a2de..5962049ee2e1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -4,72 +4,76 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.pass_context -async def java(ctx: click.Context): +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - success = await format_java(fix) + success = await format_java(ctx) if not success: click.Abort() -async def format_java(fix: bool) -> bool: +async def format_java(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") + + fix = ctx.params.get("fix_formatting") if fix: gradle_command = ["./gradlew", "spotlessApply"] else: gradle_command = ["./gradlew", "spotlessCheck", "--scan"] - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - format_container = await ( - dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") + try: + format_container = await ( + dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(gradle_command) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(gradle_command) + ) - await format_container - if fix: - await format_container.directory("/src").export(".") + await format_container + if fix: + await format_container.directory("/src").export(".") + return True - return True - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - return False + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + return False From c21302ebc484c43037a433b244a9a04faaf406a3 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 13:25:16 -0500 Subject: [PATCH 061/141] fix rest of commands to use pipelinecontext --- .../airbyte_ci/format/js/commands.py | 85 +++++++++--------- .../airbyte_ci/format/license/commands.py | 67 +++++++------- .../airbyte_ci/format/python/commands.py | 89 ++++++++++--------- 3 files changed, 124 insertions(+), 117 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index 8e24bcdc56ce..9f5a22f23dff 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -4,23 +4,25 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.pass_context -async def js(ctx: click.Context): +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - success = await format_js(fix) + success = await format_js(ctx) if not success: click.Abort() -async def format_js(fix: bool) -> bool: +async def format_js(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -29,47 +31,44 @@ async def format_js(fix: bool) -> bool: """ logger = logging.getLogger(f"format") + fix = ctx.params.get("fix_formatting") if fix: prettier_command = ["prettier", "--write", "."] else: prettier_command = ["prettier", "--check", "."] - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - format_container = await ( - dagger_client.container() - .from_("node:18.18.0-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") + + try: + format_container = await ( + dagger_client.container() + .from_("node:18.18.0-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_workdir(f"/src") - .with_exec(["npm", "install", "-g", "npm@10.1.0"]) - .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(prettier_command) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["npm", "install", "-g", "npm@10.1.0"]) + .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) + .with_exec(prettier_command) + ) - await format_container - if fix: - await format_container.directory("/src").export(".") - return True - except Exception as e: - logger.error(f"Failed to format code: {e}") - return False - - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - sys.exit(1) + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + except Exception as e: + logger.error(f"Failed to format code: {e}") + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py index eae3b56ea783..cac5053d2e1b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -6,53 +6,56 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.pass_context -async def license(ctx: click.Context): +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - - success = await format_license(fix) + success = await format_license(ctx) if not success: click.Abort() -async def format_license(fix: bool = False) -> bool: +async def format_license(ctx: ClickPipelineContext) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") + fix = ctx.params.get("fix_formatting") if fix: addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] else: addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - license_container = await ( - dagger_client.container() - .from_("golang:1.17") - .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(addlicense_command) + dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") + try: + license_container = await ( + dagger_client.container() + .from_("golang:1.17") + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), ) - - await license_container - if fix: - await license_container.directory("/src").export(".") - return True - except Exception as e: - logger.error(f"Failed to apply license: {e}") - return False + .with_workdir(f"/src") + .with_exec(addlicense_command) + ) + + await license_container + if fix: + await license_container.directory("/src").export(".") + return True + + except Exception as e: + logger.error(f"Failed to apply license: {e}") + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index 2db5d87d353f..50b6971a0cfd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -4,23 +4,24 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.pass_context -async def python(ctx: click.Context): +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - fix = ctx.obj.get("fix_formatting") - if fix is None: - raise click.UsageError("You must specify either --fix or --check") - - success = await format_python(fix) + success = await format_python(ctx) if not success: click.Abort() -async def format_python(fix: bool) -> bool: +async def format_python(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -28,46 +29,50 @@ async def format_python(fix: bool) -> bool: bool: True if the check/format succeeded, false otherwise """ logger = logging.getLogger(f"format") + fix = ctx.params.get("fix_formatting") + isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] if fix: isort_command.remove("--check-only") black_command.remove("--check") - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - format_container = await ( - dagger_client.container() - .from_("python:3.10.13-slim") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST - ), + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") + + try: + format_container = await ( + dagger_client.container() + .from_("python:3.10.13-slim") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(isort_command) - .with_exec(black_command) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST + ), + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(isort_command) + .with_exec(black_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True - await format_container - if fix: - await format_container.directory("/src").export(".") - return True - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - sys.exit(1) + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + sys.exit(1) From 80bf80032ff106ac402234a3ba26ed2ca20e73a8 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 13:36:20 -0500 Subject: [PATCH 062/141] oops, don't explicitly set the fix context param, let it come in through arguments --- .../pipelines/pipelines/airbyte_ci/format/commands.py | 2 -- .../pipelines/pipelines/airbyte_ci/format/java/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/js/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/license/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/python/commands.py | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 3aaa6a60df6a..d8900dc6d42a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -31,8 +31,6 @@ async def format(ctx: click.Context, fix: bool): from pipelines.airbyte_ci.format.license.commands import license from pipelines.airbyte_ci.format.python.commands import python - ctx.obj["fix_formatting"] = fix - if ctx.invoked_subcommand is None: # TODO: ctx.forward should forward the fix commands to the subcommands await ctx.invoke(license) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 5962049ee2e1..7e2d614b37d6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -25,7 +25,7 @@ async def java(ctx: ClickPipelineContext): async def format_java(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") - fix = ctx.params.get("fix_formatting") + fix = ctx.params["fix"] if fix: gradle_command = ["./gradlew", "spotlessApply"] else: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index 9f5a22f23dff..c35b1a7ad618 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -31,7 +31,7 @@ async def format_js(ctx: ClickPipelineContext) -> bool: """ logger = logging.getLogger(f"format") - fix = ctx.params.get("fix_formatting") + fix = ctx.params["fix"] if fix: prettier_command = ["prettier", "--write", "."] else: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py index cac5053d2e1b..03a6477f6751 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -27,7 +27,7 @@ async def format_license(ctx: ClickPipelineContext) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") - fix = ctx.params.get("fix_formatting") + fix = ctx.params["fix"] if fix: addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] else: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index 50b6971a0cfd..f2a95b542ee2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -29,7 +29,7 @@ async def format_python(ctx: ClickPipelineContext) -> bool: bool: True if the check/format succeeded, false otherwise """ logger = logging.getLogger(f"format") - fix = ctx.params.get("fix_formatting") + fix = ctx.params["fix"] isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] From d3aa826eb7ad72ed5a6032ea0f01d6b35a4929fe Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 13:56:27 -0500 Subject: [PATCH 063/141] share a dagger client when running all the commands --- .../pipelines/airbyte_ci/format/commands.py | 17 ++++++++++------- .../airbyte_ci/format/java/commands.py | 10 ++++++---- .../pipelines/airbyte_ci/format/js/commands.py | 10 ++++++---- .../airbyte_ci/format/license/commands.py | 10 ++++++---- .../airbyte_ci/format/python/commands.py | 10 ++++++---- 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index d8900dc6d42a..919b5f14cfa4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -7,9 +7,11 @@ """ import asyncclick as click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.group( cls=LazyGroup, @@ -24,16 +26,17 @@ ) @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click_merge_args_into_context_obj +@pass_pipeline_context @click_ignore_unused_kwargs -async def format(ctx: click.Context, fix: bool): +async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext, fix: bool): from pipelines.airbyte_ci.format.java.commands import java from pipelines.airbyte_ci.format.js.commands import js from pipelines.airbyte_ci.format.license.commands import license from pipelines.airbyte_ci.format.python.commands import python if ctx.invoked_subcommand is None: - # TODO: ctx.forward should forward the fix commands to the subcommands - await ctx.invoke(license) - await ctx.invoke(java) - await ctx.invoke(js) - await ctx.invoke(python) + dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") + await ctx.invoke(license, dagger_client) + await ctx.invoke(java, dagger_client) + await ctx.invoke(js, dagger_client) + await ctx.invoke(python, dagger_client) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py index 7e2d614b37d6..1532dc56d766 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Optional import asyncclick as click import dagger @@ -14,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): +async def java(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - success = await format_java(ctx) + success = await format_java(dagger_client, ctx) if not success: click.Abort() -async def format_java(ctx: ClickPipelineContext) -> bool: +async def format_java(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") fix = ctx.params["fix"] @@ -31,7 +32,8 @@ async def format_java(ctx: ClickPipelineContext) -> bool: else: gradle_command = ["./gradlew", "spotlessCheck", "--scan"] - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") try: format_container = await ( dagger_client.container() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py index c35b1a7ad618..c0fb8081af1c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Optional import asyncclick as click import dagger @@ -14,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): +async def js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - success = await format_js(ctx) + success = await format_js(dagger_client, ctx) if not success: click.Abort() -async def format_js(ctx: ClickPipelineContext) -> bool: +async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -37,7 +38,8 @@ async def format_js(ctx: ClickPipelineContext) -> bool: else: prettier_command = ["prettier", "--check", "."] - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") try: format_container = await ( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py index 03a6477f6751..15d2dc433e47 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py @@ -2,6 +2,7 @@ import logging import sys +from typing import Optional import asyncclick as click import dagger @@ -16,14 +17,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): +async def license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - success = await format_license(ctx) + success = await format_license(dagger_client, ctx) if not success: click.Abort() -async def format_license(ctx: ClickPipelineContext) -> bool: +async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") @@ -33,7 +34,8 @@ async def format_license(ctx: ClickPipelineContext) -> bool: else: addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] - dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") try: license_container = await ( dagger_client.container() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py index f2a95b542ee2..180c3ac104de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Optional import asyncclick as click import dagger @@ -14,14 +15,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): +async def python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): """Format python code via black and isort.""" - success = await format_python(ctx) + success = await format_python(dagger_client, ctx) if not success: click.Abort() -async def format_python(ctx: ClickPipelineContext) -> bool: +async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -37,7 +38,8 @@ async def format_python(ctx: ClickPipelineContext) -> bool: isort_command.remove("--check-only") black_command.remove("--check") - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") try: format_container = await ( From 7bd0ab4d59fecb634b5195802d8185bf48ccd98e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 16:29:15 -0500 Subject: [PATCH 064/141] add initial chaining structure --- .../pipelines/airbyte_ci/format/commands.py | 112 +++++++++++++++--- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 919b5f14cfa4..127b0721eca5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -6,7 +6,9 @@ Module exposing the tests command to test airbyte-ci projects. """ +from typing import Optional import asyncclick as click +import dagger from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext @@ -17,26 +19,104 @@ cls=LazyGroup, help="Commands related to formatting.", lazy_subcommands={ - "java": "pipelines.airbyte_ci.format.java.commands.java", - "js": "pipelines.airbyte_ci.format.js.commands.js", - "license": "pipelines.airbyte_ci.format.license.commands.license", - "python": "pipelines.airbyte_ci.format.python.commands.python", + # "java": "pipelines.airbyte_ci.format.java.commands.java", + # "js": "pipelines.airbyte_ci.format.js.commands.js", + # "license": "pipelines.airbyte_ci.format.license.commands.license", + # "python": "pipelines.airbyte_ci.format.python.commands.python", + # "check": "pipelines.airbyte_ci.format.commands.check", + # "fix": "pipelines.airbyte_ci.format.commands.fix", }, invoke_without_command=True, + # chain=True, ) -@click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") +# @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext, fix: bool): - from pipelines.airbyte_ci.format.java.commands import java - from pipelines.airbyte_ci.format.js.commands import js - from pipelines.airbyte_ci.format.license.commands import license - from pipelines.airbyte_ci.format.python.commands import python +async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): + pass + # from pipelines.airbyte_ci.format.java.commands import java + # from pipelines.airbyte_ci.format.js.commands import js + # from pipelines.airbyte_ci.format.license.commands import license + # from pipelines.airbyte_ci.format.python.commands import python - if ctx.invoked_subcommand is None: - dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") - await ctx.invoke(license, dagger_client) - await ctx.invoke(java, dagger_client) - await ctx.invoke(js, dagger_client) - await ctx.invoke(python, dagger_client) + # if ctx.invoked_subcommand is None: + # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") + # await ctx.invoke(license, dagger_client) + # await ctx.invoke(java, dagger_client) + # await ctx.invoke(js, dagger_client) + # await ctx.invoke(python, dagger_client) + + +@format.group(chain=True) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def check(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("check group") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking java") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking js") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking license") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking python") + + + + +@format.group(chain=True) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def fix(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fix group") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing java") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing js") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing license") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing python") From f625e258c42eee55ef42421b97192262c077112b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 16:30:54 -0500 Subject: [PATCH 065/141] use lazy imports for the top level group and clean up args --- .../pipelines/airbyte_ci/format/commands.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 127b0721eca5..57b7d584742a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -19,17 +19,11 @@ cls=LazyGroup, help="Commands related to formatting.", lazy_subcommands={ - # "java": "pipelines.airbyte_ci.format.java.commands.java", - # "js": "pipelines.airbyte_ci.format.js.commands.js", - # "license": "pipelines.airbyte_ci.format.license.commands.license", - # "python": "pipelines.airbyte_ci.format.python.commands.python", - # "check": "pipelines.airbyte_ci.format.commands.check", - # "fix": "pipelines.airbyte_ci.format.commands.fix", + "check": "pipelines.airbyte_ci.format.commands.check", + "fix": "pipelines.airbyte_ci.format.commands.fix", }, invoke_without_command=True, - # chain=True, ) -# @click.option("--fix/--check", type=bool, default=None, help="Whether to automatically fix any formatting issues detected. [required]") @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -48,7 +42,7 @@ async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): # await ctx.invoke(python, dagger_client) -@format.group(chain=True) +@click.group(chain=True) @pass_pipeline_context @click_ignore_unused_kwargs async def check(ctx: ClickPipelineContext): @@ -86,7 +80,7 @@ async def python(ctx: ClickPipelineContext): -@format.group(chain=True) +@click.group(chain=True) @pass_pipeline_context @click_ignore_unused_kwargs async def fix(ctx: ClickPipelineContext): From d78aa467ebd916391c5740ddbb121be37f3678ca Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 16:51:55 -0500 Subject: [PATCH 066/141] move invoked subcommand stuff to right area --- .../pipelines/airbyte_ci/format/commands.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 57b7d584742a..4023e4050f2a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -29,10 +29,14 @@ @click_ignore_unused_kwargs async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): pass - # from pipelines.airbyte_ci.format.java.commands import java - # from pipelines.airbyte_ci.format.js.commands import js - # from pipelines.airbyte_ci.format.license.commands import license - # from pipelines.airbyte_ci.format.python.commands import python + +@click.group(chain=True) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def check(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("check group") + # TODO: check should handle async # if ctx.invoked_subcommand is None: # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") @@ -42,12 +46,6 @@ async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): # await ctx.invoke(python, dagger_client) -@click.group(chain=True) -@pass_pipeline_context -@click_ignore_unused_kwargs -async def check(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("check group") @check.command() @pass_pipeline_context @@ -87,6 +85,13 @@ async def fix(ctx: ClickPipelineContext): """Run code format checks and fix any failures.""" print("fix group") + # if ctx.invoked_subcommand is None: + # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") + # await ctx.invoke(license, dagger_client) + # await ctx.invoke(java, dagger_client) + # await ctx.invoke(js, dagger_client) + # await ctx.invoke(python, dagger_client) + @fix.command() @pass_pipeline_context @click_ignore_unused_kwargs From 3fb30ba2e56504739a7fd3c3d7831bafba1af855 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 17:20:45 -0500 Subject: [PATCH 067/141] separate out commands and get format check java working --- .../airbyte_ci/format/check/commands.py | 57 +++++++++++ .../airbyte_ci/format/check/java/commands.py | 72 ++++++++++++++ .../format/{ => check}/js/commands.py | 0 .../format/{ => check}/license/commands.py | 0 .../format/{ => check}/python/commands.py | 0 .../pipelines/airbyte_ci/format/commands.py | 96 +------------------ .../airbyte_ci/format/fix/commands.py | 52 ++++++++++ .../format/{ => fix}/java/commands.py | 5 + .../airbyte_ci/format/fix/js/commands.py | 76 +++++++++++++++ .../airbyte_ci/format/fix/license/commands.py | 63 ++++++++++++ .../airbyte_ci/format/fix/python/commands.py | 80 ++++++++++++++++ 11 files changed, 408 insertions(+), 93 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/{ => check}/js/commands.py (100%) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/{ => check}/license/commands.py (100%) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/{ => check}/python/commands.py (100%) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/{ => fix}/java/commands.py (99%) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py new file mode 100644 index 000000000000..e8632c4bd7f9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -0,0 +1,57 @@ +from typing import Optional +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.check.java.commands import java +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +@click.group( + cls=LazyGroup, + help="Commands related to formatting.", + lazy_subcommands={ + "java": "pipelines.airbyte_ci.format.check.java.commands.java", + # "fix": "pipelines.airbyte_ci.format.fix.commands.fix", + }, + invoke_without_command=True, + chain=True, +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("check group") + # TODO: check should handle async + + # if ctx.invoked_subcommand is None: + # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") + # await ctx.invoke(license, dagger_client) + # await ctx.invoke(java, dagger_client) + # await ctx.invoke(js, dagger_client) + # await ctx.invoke(python, dagger_client) + + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking js") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking license") + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Run code format checks and fail if any checks fail.""" + print("checking python") \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py new file mode 100644 index 000000000000..4f831fafeaa0 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -0,0 +1,72 @@ +import logging +import sys +from typing import Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + +@click.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): + """Format java, groovy, and sql code via spotless.""" + + success = await check_java(ctx, dagger_client) + if not success: + click.Abort() + + +async def check_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client]) -> bool: + logger = logging.getLogger("format") + + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") + try: + format_container = await ( + dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir("/src") + .with_exec(["./gradlew", "spotlessCheck", "--scan"]) + ) + + await format_container + return True + + except dagger.ExecError as e: + logger.error("Format check failed") + logger.error(e.stderr) + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/js/commands.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/license/commands.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/python/commands.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 4023e4050f2a..4911a830fa24 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -3,7 +3,7 @@ # """ -Module exposing the tests command to test airbyte-ci projects. +Module exposing the format command. """ from typing import Optional @@ -19,8 +19,8 @@ cls=LazyGroup, help="Commands related to formatting.", lazy_subcommands={ - "check": "pipelines.airbyte_ci.format.commands.check", - "fix": "pipelines.airbyte_ci.format.commands.fix", + "check": "pipelines.airbyte_ci.format.check.commands.check", + "fix": "pipelines.airbyte_ci.format.fix.commands.fix", }, invoke_without_command=True, ) @@ -29,93 +29,3 @@ @click_ignore_unused_kwargs async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): pass - -@click.group(chain=True) -@pass_pipeline_context -@click_ignore_unused_kwargs -async def check(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("check group") - # TODO: check should handle async - - # if ctx.invoked_subcommand is None: - # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") - # await ctx.invoke(license, dagger_client) - # await ctx.invoke(java, dagger_client) - # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) - - - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking java") - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking js") - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking license") - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking python") - - - - -@click.group(chain=True) -@pass_pipeline_context -@click_ignore_unused_kwargs -async def fix(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fix group") - - # if ctx.invoked_subcommand is None: - # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") - # await ctx.invoke(license, dagger_client) - # await ctx.invoke(java, dagger_client) - # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing java") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing js") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing license") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing python") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py new file mode 100644 index 000000000000..bad6fb96c7bd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -0,0 +1,52 @@ +from typing import Optional +import asyncclick as click +import dagger +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +@click.group(chain=True) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def fix(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fix group") + + # if ctx.invoked_subcommand is None: + # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") + # await ctx.invoke(license, dagger_client) + # await ctx.invoke(java, dagger_client) + # await ctx.invoke(js, dagger_client) + # await ctx.invoke(python, dagger_client) + + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing java") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing js") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing license") + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Run code format checks and fix any failures.""" + print("fixing python") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py similarity index 99% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index 1532dc56d766..593c053d8b85 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -12,6 +12,11 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + + + + @click.command() @pass_pipeline_context @click_ignore_unused_kwargs diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py new file mode 100644 index 000000000000..c0fb8081af1c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -0,0 +1,76 @@ +import logging +import sys +from typing import Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +@click.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): + """Format yaml and json code via prettier.""" + + success = await format_js(dagger_client, ctx) + if not success: + click.Abort() + + +async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: + """Checks whether the repository is formatted correctly. + Args: + fix (bool): Whether to automatically fix any formatting issues detected. + Returns: + bool: True if the check/format succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + + fix = ctx.params["fix"] + if fix: + prettier_command = ["prettier", "--write", "."] + else: + prettier_command = ["prettier", "--check", "."] + + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") + + try: + format_container = await ( + dagger_client.container() + .from_("node:18.18.0-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["npm", "install", "-g", "npm@10.1.0"]) + .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) + .with_exec(prettier_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + except Exception as e: + logger.error(f"Failed to format code: {e}") + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py new file mode 100644 index 000000000000..15d2dc433e47 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -0,0 +1,63 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import sys +from typing import Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +@click.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): + """Add license to python and java code via addlicense.""" + success = await format_license(dagger_client, ctx) + if not success: + click.Abort() + + +async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: + license_text = "LICENSE_SHORT" + logger = logging.getLogger(f"format") + + fix = ctx.params["fix"] + if fix: + addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] + else: + addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] + + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") + try: + license_container = await ( + dagger_client.container() + .from_("golang:1.17") + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(addlicense_command) + ) + + await license_container + if fix: + await license_container.directory("/src").export(".") + return True + + except Exception as e: + logger.error(f"Failed to apply license: {e}") + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py new file mode 100644 index 000000000000..180c3ac104de --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -0,0 +1,80 @@ +import logging +import sys +from typing import Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +@click.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): + """Format python code via black and isort.""" + success = await format_python(dagger_client, ctx) + if not success: + click.Abort() + + +async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: + """Checks whether the repository is formatted correctly. + Args: + fix (bool): Whether to automatically fix any formatting issues detected. + Returns: + bool: True if the check/format succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + fix = ctx.params["fix"] + + isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] + black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] + if fix: + isort_command.remove("--check-only") + black_command.remove("--check") + + if not dagger_client: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") + + try: + format_container = await ( + dagger_client.container() + .from_("python:3.10.13-slim") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST + ), + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(isort_command) + .with_exec(black_command) + ) + + await format_container + if fix: + await format_container.directory("/src").export(".") + return True + + except dagger.ExecError as e: + logger.error("Format failed") + logger.error(e.stderr) + sys.exit(1) From ff8cf9a3689a5e3eab050e877168238e77514d25 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 17:39:11 -0500 Subject: [PATCH 068/141] get check commands and chaining going --- .../airbyte_ci/format/check/commands.py | 35 +++++-------------- .../airbyte_ci/format/check/java/commands.py | 5 ++- .../airbyte_ci/format/check/js/commands.py | 20 +++-------- .../format/check/license/commands.py | 19 +++------- .../format/check/python/commands.py | 23 ++++-------- 5 files changed, 27 insertions(+), 75 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index e8632c4bd7f9..7b8b11d3e036 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -2,7 +2,7 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.check.java.commands import java -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import LazyPassDecorator, click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext @@ -14,7 +14,9 @@ help="Commands related to formatting.", lazy_subcommands={ "java": "pipelines.airbyte_ci.format.check.java.commands.java", - # "fix": "pipelines.airbyte_ci.format.fix.commands.fix", + "js": "pipelines.airbyte_ci.format.check.js.commands.js", + "license": "pipelines.airbyte_ci.format.check.license.commands.license", + "python": "pipelines.airbyte_ci.format.check.python.commands.python", }, invoke_without_command=True, chain=True, @@ -25,33 +27,14 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fail if any checks fail.""" print("check group") - # TODO: check should handle async + # TODO: fix this client hacking + ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") + + # TODO: check should handle async # if ctx.invoked_subcommand is None: # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") # await ctx.invoke(license, dagger_client) # await ctx.invoke(java, dagger_client) # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) - - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking js") - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking license") - -@check.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Run code format checks and fail if any checks fail.""" - print("checking python") \ No newline at end of file + # await ctx.invoke(python, dagger_client) \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 4f831fafeaa0..4aea69462c65 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -22,11 +22,10 @@ async def java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] click.Abort() -async def check_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client]) -> bool: +async def check_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: logger = logging.getLogger("format") - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index c0fb8081af1c..1666bdb56e26 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -15,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Format yaml and json code via prettier.""" - success = await format_js(dagger_client, ctx) + success = await format_js(ctx, dagger_client) if not success: click.Abort() -async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -32,15 +32,7 @@ async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineCo """ logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - if fix: - prettier_command = ["prettier", "--write", "."] - else: - prettier_command = ["prettier", "--check", "."] - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") - + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() @@ -64,12 +56,10 @@ async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineCo .with_workdir(f"/src") .with_exec(["npm", "install", "-g", "npm@10.1.0"]) .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(prettier_command) + .with_exec(["prettier", "--check", "."]) ) await format_container - if fix: - await format_container.directory("/src").export(".") return True except Exception as e: logger.error(f"Failed to format code: {e}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index 15d2dc433e47..12800c0b6966 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -17,25 +17,18 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Add license to python and java code via addlicense.""" - success = await format_license(dagger_client, ctx) + success = await format_license(ctx, dagger_client) if not success: click.Abort() -async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - if fix: - addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] - else: - addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") + dagger_client = ctx.params["dagger_client"] try: license_container = await ( dagger_client.container() @@ -50,12 +43,10 @@ async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipel ), ) .with_workdir(f"/src") - .with_exec(addlicense_command) + .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."]) ) await license_container - if fix: - await license_container.directory("/src").export(".") return True except Exception as e: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 180c3ac104de..915c6775dd3a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -15,14 +15,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Format python code via black and isort.""" - success = await format_python(dagger_client, ctx) + success = await format_python(ctx, dagger_client) if not success: click.Abort() -async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -30,17 +30,8 @@ async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipeli bool: True if the check/format succeeded, false otherwise """ logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - - isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] - black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] - if fix: - isort_command.remove("--check-only") - black_command.remove("--check") - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() @@ -65,13 +56,11 @@ async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipeli ) .with_workdir(f"/src") .with_exec(["poetry", "install"]) - .with_exec(isort_command) - .with_exec(black_command) + .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."]) + .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) ) await format_container - if fix: - await format_container.directory("/src").export(".") return True except dagger.ExecError as e: From ced122b25ff2db83588c92f29352ef0f5e30efb0 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 17:49:46 -0500 Subject: [PATCH 069/141] get fix commands and chaining going --- .../airbyte_ci/format/check/commands.py | 12 +++-- .../airbyte_ci/format/check/java/commands.py | 1 + .../pipelines/airbyte_ci/format/commands.py | 2 + .../airbyte_ci/format/fix/commands.py | 49 +++++++------------ .../airbyte_ci/format/fix/java/commands.py | 25 +++------- .../airbyte_ci/format/fix/js/commands.py | 21 +++----- .../airbyte_ci/format/fix/license/commands.py | 22 +++------ .../airbyte_ci/format/fix/python/commands.py | 24 +++------ 8 files changed, 54 insertions(+), 102 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 7b8b11d3e036..5dc34461de94 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,8 +1,14 @@ from typing import Optional + import asyncclick as click import dagger from pipelines.airbyte_ci.format.check.java.commands import java -from pipelines.cli.click_decorators import LazyPassDecorator, click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import ( + LazyPassDecorator, + click_append_to_context_object, + click_ignore_unused_kwargs, + click_merge_args_into_context_obj, +) from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext @@ -26,8 +32,6 @@ @click_ignore_unused_kwargs async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fail if any checks fail.""" - print("check group") - # TODO: fix this client hacking ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") @@ -37,4 +41,4 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): # await ctx.invoke(license, dagger_client) # await ctx.invoke(java, dagger_client) # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) \ No newline at end of file + # await ctx.invoke(python, dagger_client) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 4aea69462c65..40fab70ded4d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -11,6 +11,7 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.command() @pass_pipeline_context @click_ignore_unused_kwargs diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 4911a830fa24..55e8149b973e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -7,6 +7,7 @@ """ from typing import Optional + import asyncclick as click import dagger from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj @@ -15,6 +16,7 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.group( cls=LazyGroup, help="Commands related to formatting.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index bad6fb96c7bd..2b056f0d42a0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,4 +1,5 @@ from typing import Optional + import asyncclick as click import dagger from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj @@ -8,12 +9,25 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) -@click.group(chain=True) +@click.group( + cls=LazyGroup, + help="Commands related to formatting.", + lazy_subcommands={ + "java": "pipelines.airbyte_ci.format.fix.java.commands.java", + "js": "pipelines.airbyte_ci.format.fix.js.commands.js", + "license": "pipelines.airbyte_ci.format.fix.license.commands.license", + "python": "pipelines.airbyte_ci.format.fix.python.commands.python", + }, + invoke_without_command=True, + chain=True, +) +@click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def fix(ctx: ClickPipelineContext): +async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fix any failures.""" - print("fix group") + # TODO: fix this client hacking + ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") # if ctx.invoked_subcommand is None: # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") @@ -21,32 +35,3 @@ async def fix(ctx: ClickPipelineContext): # await ctx.invoke(java, dagger_client) # await ctx.invoke(js, dagger_client) # await ctx.invoke(python, dagger_client) - - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing java") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing js") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing license") - -@fix.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Run code format checks and fix any failures.""" - print("fixing python") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index 593c053d8b85..bd29421ef9f2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -12,33 +12,21 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - - - - - @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def java(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Format java, groovy, and sql code via spotless.""" - success = await format_java(dagger_client, ctx) + success = await format_java(ctx, dagger_client) if not success: click.Abort() -async def format_java(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: logger = logging.getLogger("format") - fix = ctx.params["fix"] - if fix: - gradle_command = ["./gradlew", "spotlessApply"] - else: - gradle_command = ["./gradlew", "spotlessCheck", "--scan"] - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Java") + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() @@ -72,12 +60,11 @@ async def format_java(dagger_client: Optional[dagger.Client], ctx: ClickPipeline ), ) .with_workdir(f"/src") - .with_exec(gradle_command) + .with_exec(["./gradlew", "spotlessApply"]) ) await format_container - if fix: - await format_container.directory("/src").export(".") + await format_container.directory("/src").export(".") return True except dagger.ExecError as e: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index c0fb8081af1c..65fbfa65315f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -15,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Format yaml and json code via prettier.""" - success = await format_js(dagger_client, ctx) + success = await format_js(ctx, dagger_client) if not success: click.Abort() -async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client]) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -32,15 +32,7 @@ async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineCo """ logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - if fix: - prettier_command = ["prettier", "--write", "."] - else: - prettier_command = ["prettier", "--check", "."] - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Yaml and Json") - + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() @@ -64,12 +56,11 @@ async def format_js(dagger_client: Optional[dagger.Client], ctx: ClickPipelineCo .with_workdir(f"/src") .with_exec(["npm", "install", "-g", "npm@10.1.0"]) .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(prettier_command) + .with_exec(["prettier", "--write", "."]) ) await format_container - if fix: - await format_container.directory("/src").export(".") + await format_container.directory("/src").export(".") return True except Exception as e: logger.error(f"Failed to format code: {e}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index 15d2dc433e47..a3323f98e149 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -17,25 +17,18 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Add license to python and java code via addlicense.""" - success = await format_license(dagger_client, ctx) + success = await format_license(ctx, dagger_client) if not success: click.Abort() -async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - if fix: - addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."] - else: - addlicense_command = ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."] - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format License") + dagger_client = ctx.params["dagger_client"] try: license_container = await ( dagger_client.container() @@ -49,13 +42,12 @@ async def format_license(dagger_client: Optional[dagger.Client], ctx: ClickPipel exclude=DEFAULT_FORMAT_IGNORE_LIST, ), ) - .with_workdir(f"/src") - .with_exec(addlicense_command) + .with_workdir("/src") + .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."]) ) await license_container - if fix: - await license_container.directory("/src").export(".") + await license_container.directory("/src").export(".") return True except Exception as e: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index 180c3ac104de..babe8498a8ac 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -15,14 +15,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext): +async def python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): """Format python code via black and isort.""" - success = await format_python(dagger_client, ctx) + success = await format_python(ctx, dagger_client) if not success: click.Abort() -async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipelineContext) -> bool: +async def format_python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. @@ -30,17 +30,8 @@ async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipeli bool: True if the check/format succeeded, false otherwise """ logger = logging.getLogger(f"format") - fix = ctx.params["fix"] - - isort_command = ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] - black_command = ["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."] - if fix: - isort_command.remove("--check-only") - black_command.remove("--check") - - if not dagger_client: - dagger_client = await ctx.get_dagger_client(pipeline_name="Format Python") + dagger_client = ctx.params["dagger_client"] try: format_container = await ( dagger_client.container() @@ -65,13 +56,12 @@ async def format_python(dagger_client: Optional[dagger.Client], ctx: ClickPipeli ) .with_workdir(f"/src") .with_exec(["poetry", "install"]) - .with_exec(isort_command) - .with_exec(black_command) + .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "."]) + .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "."]) ) await format_container - if fix: - await format_container.directory("/src").export(".") + await format_container.directory("/src").export(".") return True except dagger.ExecError as e: From c2fe2b9cb8910e2f168e2970ae621be74fd14929 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Tue, 31 Oct 2023 18:11:16 -0500 Subject: [PATCH 070/141] improve help --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/commands.py | 1 - .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 5dc34461de94..1883b08f3cbc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -17,7 +17,7 @@ @click.group( cls=LazyGroup, - help="Commands related to formatting.", + help="Run code format checks and fail if any checks fail.", lazy_subcommands={ "java": "pipelines.airbyte_ci.format.check.java.commands.java", "js": "pipelines.airbyte_ci.format.check.js.commands.js", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 55e8149b973e..6d6fe5d90f6b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -24,7 +24,6 @@ "check": "pipelines.airbyte_ci.format.check.commands.check", "fix": "pipelines.airbyte_ci.format.fix.commands.fix", }, - invoke_without_command=True, ) @click_merge_args_into_context_obj @pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 2b056f0d42a0..6ec584d167ea 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -11,7 +11,7 @@ @click.group( cls=LazyGroup, - help="Commands related to formatting.", + help="Run code format checks and fix any failures.", lazy_subcommands={ "java": "pipelines.airbyte_ci.format.fix.java.commands.java", "js": "pipelines.airbyte_ci.format.fix.js.commands.js", @@ -29,7 +29,8 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): # TODO: fix this client hacking ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") - # if ctx.invoked_subcommand is None: + if ctx.invoked_subcommand is None: + print("no args invoked") # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") # await ctx.invoke(license, dagger_client) # await ctx.invoke(java, dagger_client) From faddea3bd41160115d95a108172586cf5bac646c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 15:49:46 -0500 Subject: [PATCH 071/141] remove dagger client param --- .../pipelines/airbyte_ci/format/check/java/commands.py | 7 +++---- .../pipelines/airbyte_ci/format/check/js/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/check/license/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/check/python/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/fix/java/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/fix/js/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/fix/license/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/fix/python/commands.py | 6 +++--- 8 files changed, 24 insertions(+), 25 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 40fab70ded4d..34df0df31e15 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -11,19 +11,18 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - success = await check_java(ctx, dagger_client) + success = await check_java(ctx) if not success: click.Abort() -async def check_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def check_java(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") dagger_client = ctx.params["dagger_client"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index 1666bdb56e26..52454253d4fa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -15,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - success = await format_js(ctx, dagger_client) + success = await format_js(ctx) if not success: click.Abort() -async def format_js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_js(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index 12800c0b6966..b15d8ba1f8e1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -17,14 +17,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - success = await format_license(ctx, dagger_client) + success = await format_license(ctx) if not success: click.Abort() -async def format_license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_license(ctx: ClickPipelineContext) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 915c6775dd3a..79e5b585020e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -15,14 +15,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - success = await format_python(ctx, dagger_client) + success = await format_python(ctx) if not success: click.Abort() -async def format_python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_python(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index bd29421ef9f2..fe84aec84c78 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -15,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - success = await format_java(ctx, dagger_client) + success = await format_java(ctx) if not success: click.Abort() -async def format_java(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_java(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") dagger_client = ctx.params["dagger_client"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 65fbfa65315f..2672bdecc741 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -15,15 +15,15 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - success = await format_js(ctx, dagger_client) + success = await format_js(ctx) if not success: click.Abort() -async def format_js(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client]) -> bool: +async def format_js(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index a3323f98e149..7c3910e53d10 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -17,14 +17,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - success = await format_license(ctx, dagger_client) + success = await format_license(ctx) if not success: click.Abort() -async def format_license(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_license(ctx: ClickPipelineContext) -> bool: license_text = "LICENSE_SHORT" logger = logging.getLogger(f"format") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index babe8498a8ac..e4fdf6148e4d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -15,14 +15,14 @@ @click.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None): +async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - success = await format_python(ctx, dagger_client) + success = await format_python(ctx) if not success: click.Abort() -async def format_python(ctx: ClickPipelineContext, dagger_client: Optional[dagger.Client] = None) -> bool: +async def format_python(ctx: ClickPipelineContext) -> bool: """Checks whether the repository is formatted correctly. Args: fix (bool): Whether to automatically fix any formatting issues detected. From 9ac7cc0e3d6db4e7aba228b65fab14edb828690e Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 15:56:28 -0500 Subject: [PATCH 072/141] fix pass pipeline context import --- .../pipelines/airbyte_ci/format/check/java/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/check/js/commands.py | 7 ++----- .../pipelines/airbyte_ci/format/check/license/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/check/python/commands.py | 6 ++---- .../pipelines/pipelines/airbyte_ci/format/commands.py | 6 ++---- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/fix/java/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/fix/js/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/fix/license/commands.py | 6 ++---- .../pipelines/airbyte_ci/format/fix/python/commands.py | 6 ++---- 10 files changed, 20 insertions(+), 41 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 34df0df31e15..19fef7ad9a2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -5,11 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() @pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index 52454253d4fa..aad04ccf880a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -5,12 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() @pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index b15d8ba1f8e1..c7968242de01 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -7,11 +7,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 79e5b585020e..383ee7ca8bc3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -5,11 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 6d6fe5d90f6b..2ef9ca80e7bf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -10,11 +10,9 @@ import asyncclick as click import dagger -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.group( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 6ec584d167ea..c657be038c8e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -2,11 +2,9 @@ import asyncclick as click import dagger -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.group( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index fe84aec84c78..c232381bd01c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -5,11 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 2672bdecc741..0b0b2b39aff2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -5,11 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index 7c3910e53d10..ef5b03998414 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -7,11 +7,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index e4fdf6148e4d..ab926dba1fce 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -5,11 +5,9 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() From b11b8dbf403612422c7d718b99cb4f747a20bca0 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 16:07:13 -0500 Subject: [PATCH 073/141] add licenses to new files --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 ++ .../pipelines/airbyte_ci/format/check/java/commands.py | 3 +++ .../pipelines/pipelines/airbyte_ci/format/check/js/commands.py | 3 +++ .../pipelines/airbyte_ci/format/check/python/commands.py | 2 ++ .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 2 ++ .../pipelines/pipelines/airbyte_ci/format/fix/java/commands.py | 2 ++ .../pipelines/pipelines/airbyte_ci/format/fix/js/commands.py | 2 ++ .../pipelines/airbyte_ci/format/fix/python/commands.py | 2 ++ 8 files changed, 18 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 1883b08f3cbc..6380007060cc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Optional import asyncclick as click diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 19fef7ad9a2f..2eae89105abd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional @@ -9,6 +11,7 @@ from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + @click.command() @pass_pipeline_context @click_ignore_unused_kwargs diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index aad04ccf880a..90e2b9eceecd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional @@ -9,6 +11,7 @@ from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + @click.command() @pass_pipeline_context @click_ignore_unused_kwargs diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 383ee7ca8bc3..1ec1d67599ca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index c657be038c8e..e92f6ed84e5d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Optional import asyncclick as click diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index c232381bd01c..49ad73a16f6c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 0b0b2b39aff2..a5b9bda15020 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index ab926dba1fce..f3880a7ecd31 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional From fc5e677073a1fd2ef7be2d7b79e2be045743ee16 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 16:54:10 -0500 Subject: [PATCH 074/141] readme and version bump --- airbyte-ci/connectors/pipelines/README.md | 5 +++-- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 80373e9ec393..1d444afc53f2 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -408,9 +408,10 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting | | 2.5.5 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | -| 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | -| 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | +| 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | +| 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | | 2.5.2 | [#31871](https://github.com/airbytehq/airbyte/pull/31871) | Deactivate PR comments, add HTML report links to the PR status when its ready. | | 2.5.1 | [#31774](https://github.com/airbytehq/airbyte/pull/31774) | Add a docker configuration check on `airbyte-ci` startup. | | 2.5.0 | [#31766](https://github.com/airbytehq/airbyte/pull/31766) | Support local connectors secrets. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 36336f46a501..afb023d33e14 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.5.5" +version = "2.6.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 2074bad0befb323a75c7910e3245173b3b2dca2c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Thu, 2 Nov 2023 17:05:55 -0500 Subject: [PATCH 075/141] remove code formatting checks from connector test --- airbyte-ci/connectors/pipelines/README.md | 3 +- .../airbyte_ci/connectors/test/pipeline.py | 28 +---------- .../test/steps/python_connectors.py | 48 ------------------- 3 files changed, 3 insertions(+), 76 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 1d444afc53f2..91326f50eb3a 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -205,7 +205,6 @@ flowchart TD entrypoint[[For each selected connector]] subgraph static ["Static code analysis"] qa[Run QA checks] - fmt[Run code format checks] sem["Check version follows semantic versionning"] incr["Check version is incremented"] metadata_validation["Run metadata validation on metadata.yaml"] @@ -408,7 +407,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting | +| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | | 2.5.5 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | | 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | | 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py index 930a392cb09f..df5a4d28888c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -21,12 +21,7 @@ ConnectorLanguage.PYTHON: python_connectors.run_all_tests, ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, ConnectorLanguage.JAVA: java_connectors.run_all_tests, - }, - "run_code_format_checks": { - ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, - ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, - # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks - }, + } } @@ -65,22 +60,6 @@ async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: return [await QaChecks(context).run()] -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format checks according to the connector language. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the code format checks steps. - """ - if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language): - return await _run_code_format_checks(context) - else: - context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!") - return [] - - async def run_all_tests(context: ConnectorContext) -> List[StepResult]: """Run all the tests steps according to the connector language. @@ -111,10 +90,7 @@ async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyi async with semaphore: async with context: async with asyncer.create_task_group() as task_group: - tasks = [ - task_group.soonify(run_all_tests)(context), - task_group.soonify(run_code_format_checks)(context), - ] + tasks = [task_group.soonify(run_all_tests)(context)] if not context.code_tests_only: tasks += [ task_group.soonify(run_metadata_validation)(context), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index 536f8ff07421..ac1b3c9f82ea 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -19,42 +19,6 @@ from pipelines.models.steps import Step, StepResult, StepStatus -class CodeFormatChecks(Step): - """A step to run the code format checks on a Python connector using Black, Isort and Flake.""" - - title = "Code format checks" - - RUN_BLACK_CMD = ["python", "-m", "black", f"--config=/{PYPROJECT_TOML_FILE_PATH}", "--check", "."] - RUN_ISORT_CMD = ["python", "-m", "isort", f"--settings-file=/{PYPROJECT_TOML_FILE_PATH}", "--check-only", "--diff", "."] - RUN_FLAKE_CMD = ["python", "-m", "pflake8", f"--config=/{PYPROJECT_TOML_FILE_PATH}", "."] - - async def _run(self) -> StepResult: - """Run a code format check on the container source code. - - We call black, isort and flake commands: - - Black formats the code: fails if the code is not formatted. - - Isort checks the import orders: fails if the import are not properly ordered. - - Flake enforces style-guides: fails if the style-guide is not followed. - - Args: - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. - step (Step): The step in which the code format checks are run. Defaults to Step.CODE_FORMAT_CHECKS - Returns: - StepResult: Failure or success of the code format checks with stdout and stderr. - """ - connector_under_test = pipelines.dagger.actions.python.common.with_python_connector_source(self.context) - - formatter = ( - connector_under_test.with_exec(["echo", "Running black"]) - .with_exec(self.RUN_BLACK_CMD) - .with_exec(["echo", "Running Isort"]) - .with_exec(self.RUN_ISORT_CMD) - .with_exec(["echo", "Running Flake"]) - .with_exec(self.RUN_FLAKE_CMD) - ) - return await self.get_step_result(formatter) - - class PytestStep(Step, ABC): """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" @@ -230,15 +194,3 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: ] return step_results + [task.value for task in tasks] - - -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format check steps for Python connectors. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: Results of the code format checks. - """ - return [await CodeFormatChecks(context).run()] From f7128796fc950402cdc8e91b50979aabdb47b7c6 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 08:42:24 -0500 Subject: [PATCH 076/141] fix commands should always exit 0 --- .../airbyte_ci/format/fix/java/commands.py | 85 ++++++++----------- .../airbyte_ci/format/fix/js/commands.py | 71 ++++++---------- .../airbyte_ci/format/fix/license/commands.py | 47 ++++------ .../airbyte_ci/format/fix/python/commands.py | 76 ++++++----------- 4 files changed, 105 insertions(+), 174 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index 49ad73a16f6c..e723d96e9fd5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -17,57 +17,42 @@ @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - - success = await format_java(ctx) - if not success: - click.Abort() - - -async def format_java(ctx: ClickPipelineContext) -> bool: - logger = logging.getLogger("format") - dagger_client = ctx.params["dagger_client"] - try: - format_container = await ( - dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), + + format_container = await ( + dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_workdir(f"/src") - .with_exec(["./gradlew", "spotlessApply"]) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["./gradlew", "spotlessApply"]) + ) - await format_container - await format_container.directory("/src").export(".") - return True - - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - return False + await format_container + await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index a5b9bda15020..22eebfc70bf7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import logging import sys from typing import Optional @@ -18,50 +16,31 @@ async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - success = await format_js(ctx) - if not success: - click.Abort() - - -async def format_js(ctx: ClickPipelineContext) -> bool: - """Checks whether the repository is formatted correctly. - Args: - fix (bool): Whether to automatically fix any formatting issues detected. - Returns: - bool: True if the check/format succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - dagger_client = ctx.params["dagger_client"] - try: - format_container = await ( - dagger_client.container() - .from_("node:18.18.0-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) + format_container = await ( + dagger_client.container() + .from_("node:18.18.0-slim") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(["npm", "install", "-g", "npm@10.1.0"]) - .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(["prettier", "--write", "."]) ) - - await format_container - await format_container.directory("/src").export(".") - return True - except Exception as e: - logger.error(f"Failed to format code: {e}") - return False + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir(f"/src") + .with_exec(["npm", "install", "-g", "npm@10.1.0"]) + .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) + .with_exec(["prettier", "--write", "."]) + ) + + await format_container + await format_container.directory("/src").export(".") \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index ef5b03998414..e257939a3768 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -17,37 +17,24 @@ @click_ignore_unused_kwargs async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - success = await format_license(ctx) - if not success: - click.Abort() - - -async def format_license(ctx: ClickPipelineContext) -> bool: - license_text = "LICENSE_SHORT" - logger = logging.getLogger(f"format") + license_file = "LICENSE_SHORT" dagger_client = ctx.params["dagger_client"] - try: - license_container = await ( - dagger_client.container() - .from_("golang:1.17") - .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir("/src") - .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "."]) + license_container = await ( + dagger_client.container() + .from_("golang:1.17") + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), ) + .with_workdir("/src") + .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_file, "."]) + ) - await license_container - await license_container.directory("/src").export(".") - return True - - except Exception as e: - logger.error(f"Failed to apply license: {e}") - return False + await license_container + await license_container.directory("/src").export(".") \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index f3880a7ecd31..98a779c8e51c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -17,54 +17,34 @@ @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - success = await format_python(ctx) - if not success: - click.Abort() - - -async def format_python(ctx: ClickPipelineContext) -> bool: - """Checks whether the repository is formatted correctly. - Args: - fix (bool): Whether to automatically fix any formatting issues detected. - Returns: - bool: True if the check/format succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - dagger_client = ctx.params["dagger_client"] - try: - format_container = await ( - dagger_client.container() - .from_("python:3.10.13-slim") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST - ), + + format_container = await ( + dagger_client.container() + .from_("python:3.10.13-slim") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "."]) - .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "."]) ) - - await format_container - await format_container.directory("/src").export(".") - return True - - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - sys.exit(1) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST + ), + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "."]) + .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "."]) + ) + + await format_container + await format_container.directory("/src").export(".") \ No newline at end of file From d27082c9d1214ca7047f86e781880a43e11aba76 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 09:03:53 -0500 Subject: [PATCH 077/141] potentially temp: give check command sys.exit(1) --- .../pipelines/airbyte_ci/format/check/java/commands.py | 3 ++- .../pipelines/airbyte_ci/format/check/js/commands.py | 7 ++++--- .../pipelines/airbyte_ci/format/check/license/commands.py | 7 ++++--- .../pipelines/airbyte_ci/format/check/python/commands.py | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 2eae89105abd..1b3a21085008 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -20,7 +20,7 @@ async def java(ctx: ClickPipelineContext): success = await check_java(ctx) if not success: - click.Abort() + sys.exit(1) async def check_java(ctx: ClickPipelineContext) -> bool: @@ -66,6 +66,7 @@ async def check_java(ctx: ClickPipelineContext) -> bool: await format_container return True + except dagger.ExecError as e: logger.error("Format check failed") logger.error(e.stderr) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index 90e2b9eceecd..8bdf2872144b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -20,7 +20,7 @@ async def js(ctx: ClickPipelineContext): success = await format_js(ctx) if not success: - click.Abort() + sys.exit(1) async def format_js(ctx: ClickPipelineContext) -> bool: @@ -61,6 +61,7 @@ async def format_js(ctx: ClickPipelineContext) -> bool: await format_container return True - except Exception as e: - logger.error(f"Failed to format code: {e}") + except dagger.ExecError as e: + logger.error(f"Failed to format code") + logger.error(e.stderr) return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index c7968242de01..03cb19f18f44 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -19,7 +19,7 @@ async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" success = await format_license(ctx) if not success: - click.Abort() + sys.exit(1) async def format_license(ctx: ClickPipelineContext) -> bool: @@ -47,6 +47,7 @@ async def format_license(ctx: ClickPipelineContext) -> bool: await license_container return True - except Exception as e: - logger.error(f"Failed to apply license: {e}") + except dagger.ExecError as e: + logger.error("Failed to apply license") + logger.error(e.stderr) return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 1ec1d67599ca..087a9168ce0b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -19,7 +19,7 @@ async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" success = await format_python(ctx) if not success: - click.Abort() + sys.exit(1) async def format_python(ctx: ClickPipelineContext) -> bool: @@ -66,4 +66,4 @@ async def format_python(ctx: ClickPipelineContext) -> bool: except dagger.ExecError as e: logger.error("Format failed") logger.error(e.stderr) - sys.exit(1) + return False From c00f03df50284cc4ca4080a42af719e4895af281 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 09:05:05 -0500 Subject: [PATCH 078/141] run via chaining for now --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 42129c4652f2..8f45f64d5bdb 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -51,7 +51,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format --check" + subcommand: "format check java js license python" notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" From c9da69fe32cf7e4ea3e8e80e07edef12881d86ee Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 10:29:14 -0500 Subject: [PATCH 079/141] format --- .../pipelines/airbyte_ci/format/check/java/commands.py | 1 - .../pipelines/airbyte_ci/format/fix/js/commands.py | 2 +- .../pipelines/airbyte_ci/format/fix/license/commands.py | 2 +- .../pipelines/airbyte_ci/format/fix/python/commands.py | 6 ++---- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 1b3a21085008..7069683610f3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -66,7 +66,6 @@ async def check_java(ctx: ClickPipelineContext) -> bool: await format_container return True - except dagger.ExecError as e: logger.error("Format check failed") logger.error(e.stderr) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 22eebfc70bf7..3451445f2253 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -43,4 +43,4 @@ async def js(ctx: ClickPipelineContext): ) await format_container - await format_container.directory("/src").export(".") \ No newline at end of file + await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index e257939a3768..b386740fc06c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -37,4 +37,4 @@ async def license(ctx: ClickPipelineContext): ) await license_container - await license_container.directory("/src").export(".") \ No newline at end of file + await license_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index 98a779c8e51c..dbcc1fd980d0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -36,9 +36,7 @@ async def python(ctx: ClickPipelineContext): ) .with_mounted_directory( "/src", - dagger_client.host().directory( - ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST - ), + dagger_client.host().directory(".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST), ) .with_workdir(f"/src") .with_exec(["poetry", "install"]) @@ -47,4 +45,4 @@ async def python(ctx: ClickPipelineContext): ) await format_container - await format_container.directory("/src").export(".") \ No newline at end of file + await format_container.directory("/src").export(".") From 58eca7a4c7babb23f6e0eb256dd8ab6dcd810548 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 10:45:07 -0500 Subject: [PATCH 080/141] bring back bulk commands and use in CI --- .github/workflows/format.yml | 2 +- .../airbyte_ci/format/check/commands.py | 19 ++++++++++++------- .../airbyte_ci/format/fix/commands.py | 17 +++++++++++------ .../airbyte_ci/format/fix/js/commands.py | 2 ++ 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8f45f64d5bdb..e5047ebfdf73 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -51,7 +51,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format check java js license python" + subcommand: "format check" notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 6380007060cc..80220df2f9a1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -37,10 +37,15 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): # TODO: fix this client hacking ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") - # TODO: check should handle async - # if ctx.invoked_subcommand is None: - # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") - # await ctx.invoke(license, dagger_client) - # await ctx.invoke(java, dagger_client) - # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) + if ctx.invoked_subcommand is None: + from pipelines.airbyte_ci.format.check.java.commands import java + from pipelines.airbyte_ci.format.check.js.commands import js + from pipelines.airbyte_ci.format.check.license.commands import license + from pipelines.airbyte_ci.format.check.python.commands import python + + print("Running all checks...") + + # await ctx.invoke(java) + await ctx.invoke(js) + await ctx.invoke(license) + await ctx.invoke(python) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index e92f6ed84e5d..bec210aff3d1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -30,9 +30,14 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") if ctx.invoked_subcommand is None: - print("no args invoked") - # dagger_client = await pipeline_ctx.get_dagger_client(pipeline_name="Format All Files") - # await ctx.invoke(license, dagger_client) - # await ctx.invoke(java, dagger_client) - # await ctx.invoke(js, dagger_client) - # await ctx.invoke(python, dagger_client) + from pipelines.airbyte_ci.format.fix.java.commands import java + from pipelines.airbyte_ci.format.fix.js.commands import js + from pipelines.airbyte_ci.format.fix.license.commands import license + from pipelines.airbyte_ci.format.fix.python.commands import python + + print("Running all formatters...") + + await ctx.invoke(java) + await ctx.invoke(js) + await ctx.invoke(license) + await ctx.invoke(python) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 3451445f2253..cd245c8c5dd9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging import sys from typing import Optional From 903b04c01b46faefa5872474a0688d0564f180cf Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 10:50:29 -0500 Subject: [PATCH 081/141] fix java formatting failure, and also speed it up significantly --- build.gradle | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 6ff8c4d2b84e..f454cba4d79e 100644 --- a/build.gradle +++ b/build.gradle @@ -111,10 +111,12 @@ rootProject.tasks.named('clean').configure { allprojects { def createFormatTarget = { pattern -> - // Ignore new build files as well as the ones ignored when running this through airbyte-ci - ArrayList excludes = ['**/build'] + ArrayList excludes = [ + '**/build', // Ignore new build files as well as the ones ignored when running this through airbyte-ci + '.gradle' // Ignore the gradle version that is downloaded in the container + ] // Build the FileTree. - return fileTree(dir: projectDir, include: pattern, exclude: ['**/build']) + return fileTree(dir: projectDir, include: pattern, exclude: excludes) } // Apply spotless formatting. From bd09adf612e09107fcc3c25b4936dcfd80cd6152 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 15:57:53 -0500 Subject: [PATCH 082/141] re-add logic to remove spotlessCheck from gradle check --- build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index f454cba4d79e..6b7d6b8de733 100644 --- a/build.gradle +++ b/build.gradle @@ -147,6 +147,10 @@ allprojects { } } } +// TODO: remove this once the CI routinely enforces formatting checks. +tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { + enabled = false +} def getCDKTargetVersion() { def props = new Properties() From 7a6e5c3dea3021e895814a656278959d986f6610 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Fri, 3 Nov 2023 16:03:23 -0500 Subject: [PATCH 083/141] check java --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 80220df2f9a1..0cd06a3f4e2b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -45,7 +45,7 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): print("Running all checks...") - # await ctx.invoke(java) + await ctx.invoke(java) await ctx.invoke(js) await ctx.invoke(license) await ctx.invoke(python) From 73fb20cb388341b4fa8886eff60b463f2429b19c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 12:50:15 -0600 Subject: [PATCH 084/141] add readme info --- airbyte-ci/connectors/pipelines/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 91326f50eb3a..0e64d551df2e 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -100,6 +100,9 @@ At this point you can run `airbyte-ci` commands. - [`connectors bump_version` command](#connectors-bump_version) - [`connectors upgrade_base_image` command](#connectors-upgrade_base_image) - [`connectors migrate_to_base_image` command](#connectors-migrate_to_base_image) +- [`format` command subgroup](#format-subgroup) + * [`format check` command](#format-check-command) + * [`format fix` command](#format-fix-command) - [`metadata` command subgroup](#metadata-command-subgroup) - [`metadata validate` command](#metadata-validate-command) * [Example](#example) @@ -368,6 +371,25 @@ Migrate source-openweather to use the base image: `airbyte-ci connectors --name= | --------------------- | ----------------------------------------------------------- | | `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | +### `format` command subgroup + +Available commands: +* `airbyte-ci format check` +* `airbyte-ci format fix` + +### `format check` command + +This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix`. + +Running `airbyte-ci format check` will run checks on all different types of code. Run `airbyte-ci format check --help` for subcommands to check formatting for only certain types of files. + +### `format fix` command + +This command runs formatting checks and reformats any code that would be reformatted, so it's recommended to stage changes you might have before running this command. + +Running `airbyte-ci format fix` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. + + ### `metadata` command subgroup Available commands: From ab94ab5c472b5d76ae3904cbd5786fe5a08a0f1d Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 12:52:22 -0600 Subject: [PATCH 085/141] add examples --- airbyte-ci/connectors/pipelines/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 0e64d551df2e..b4795ba58a64 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -377,6 +377,10 @@ Available commands: * `airbyte-ci format check` * `airbyte-ci format fix` +### Examples +- Check for formatting errors in the repository: `airbyte-ci format check` +- Fix formatting for only python files: `airbyte-ci format fix python` + ### `format check` command This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix`. From 930b053f8ee2f42c56da51c642886ff012ebd342 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 13:28:32 -0600 Subject: [PATCH 086/141] run checks concurrently --- .../pipelines/airbyte_ci/format/check/commands.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 0cd06a3f4e2b..7a1312ee2bcf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -2,6 +2,7 @@ from typing import Optional +import anyio import asyncclick as click import dagger from pipelines.airbyte_ci.format.check.java.commands import java @@ -45,7 +46,8 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): print("Running all checks...") - await ctx.invoke(java) - await ctx.invoke(js) - await ctx.invoke(license) - await ctx.invoke(python) + async with anyio.create_task_group() as check_group: + check_group.start_soon(ctx.invoke, java) + check_group.start_soon(ctx.invoke, js) + check_group.start_soon(ctx.invoke, license) + check_group.start_soon(ctx.invoke, python) From 39ae84411f99fb10ce6cd6073130800358accca1 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 13:39:48 -0600 Subject: [PATCH 087/141] remove formatting stuff from gradleTask base --- .../connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 7dedcf610959..5221636dd281 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -77,7 +77,6 @@ async def _run(self) -> StepResult: "buildSrc", "tools/bin/build_image.sh", "tools/lib/lib.sh", - "tools/gradle/codestyle", "pyproject.toml", ] + self.build_include @@ -85,8 +84,6 @@ async def _run(self) -> StepResult: "docker", # required by :integrationTestJava. "findutils", # gradle requires xargs, which is shipped in findutils. "jq", # required by :acceptance-test-harness to inspect docker images. - "npm", # required by :format. - "python3.11-pip", # required by :format. "rsync", # required for gradle cache synchronization. ] From d79f05692fe1f84f7056e759eeaeccf70a9cf07d Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 13:44:29 -0600 Subject: [PATCH 088/141] java: feedback on includes --- .../pipelines/airbyte_ci/format/check/java/commands.py | 6 +++--- .../pipelines/airbyte_ci/format/fix/java/commands.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 7069683610f3..87af9b0a0aa0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -8,6 +8,7 @@ import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import click_ignore_unused_kwargs +from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -30,7 +31,7 @@ async def check_java(ctx: ClickPipelineContext) -> bool: try: format_container = await ( dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") + .from_(AMAZONCORRETTO_IMAGE) .with_exec( sh_dash_c( [ @@ -48,9 +49,8 @@ async def check_java(ctx: ClickPipelineContext) -> bool: "**/*.sql", "**/*.gradle", "gradlew", - "gradlew.bat", "gradle", - "**/deps.toml", + "deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index e723d96e9fd5..59c0d7937f9e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -39,9 +39,8 @@ async def java(ctx: ClickPipelineContext): "**/*.sql", "**/*.gradle", "gradlew", - "gradlew.bat", "gradle", - "**/deps.toml", + "deps.toml", "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", From ee16e72b73ef30c72a3c15f0884a3c9a9eb4ad5b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 13:59:00 -0600 Subject: [PATCH 089/141] don't await containers as we create them --- .../pipelines/airbyte_ci/format/check/java/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/check/js/commands.py | 2 +- .../pipelines/airbyte_ci/format/check/license/commands.py | 2 +- .../pipelines/airbyte_ci/format/check/python/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/fix/java/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/fix/js/commands.py | 2 +- .../pipelines/airbyte_ci/format/fix/license/commands.py | 2 +- .../pipelines/airbyte_ci/format/fix/python/commands.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index 87af9b0a0aa0..f6aab9064eed 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -29,7 +29,7 @@ async def check_java(ctx: ClickPipelineContext) -> bool: dagger_client = ctx.params["dagger_client"] try: - format_container = await ( + format_container = ( dagger_client.container() .from_(AMAZONCORRETTO_IMAGE) .with_exec( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index 8bdf2872144b..dc17fcf38bf9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -34,7 +34,7 @@ async def format_js(ctx: ClickPipelineContext) -> bool: dagger_client = ctx.params["dagger_client"] try: - format_container = await ( + format_container = ( dagger_client.container() .from_("node:18.18.0-slim") .with_exec( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index 03cb19f18f44..365ca07bd37e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -28,7 +28,7 @@ async def format_license(ctx: ClickPipelineContext) -> bool: dagger_client = ctx.params["dagger_client"] try: - license_container = await ( + license_container = ( dagger_client.container() .from_("golang:1.17") .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 087a9168ce0b..e931e62a54dd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -33,7 +33,7 @@ async def format_python(ctx: ClickPipelineContext) -> bool: dagger_client = ctx.params["dagger_client"] try: - format_container = await ( + format_container = ( dagger_client.container() .from_("python:3.10.13-slim") .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index 59c0d7937f9e..185cae7b26b5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -19,7 +19,7 @@ async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" dagger_client = ctx.params["dagger_client"] - format_container = await ( + format_container = ( dagger_client.container() .from_("openjdk:17.0.1-jdk-slim") .with_exec( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index cd245c8c5dd9..c0446967c413 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -19,7 +19,7 @@ async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" dagger_client = ctx.params["dagger_client"] - format_container = await ( + format_container = ( dagger_client.container() .from_("node:18.18.0-slim") .with_exec( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index b386740fc06c..80aff99a4251 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -20,7 +20,7 @@ async def license(ctx: ClickPipelineContext): license_file = "LICENSE_SHORT" dagger_client = ctx.params["dagger_client"] - license_container = await ( + license_container = ( dagger_client.container() .from_("golang:1.17") .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index dbcc1fd980d0..6446dea780e2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -19,7 +19,7 @@ async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" dagger_client = ctx.params["dagger_client"] - format_container = await ( + format_container = ( dagger_client.container() .from_("python:3.10.13-slim") .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") From 3276d485dc227542c7e4f907af65e936f02cf001 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 14:18:37 -0600 Subject: [PATCH 090/141] refactor --- .../airbyte_ci/format/check/java/commands.py | 66 +++++++++---------- .../airbyte_ci/format/check/js/commands.py | 47 +++++++------ .../format/check/license/commands.py | 34 +++++----- .../format/check/python/commands.py | 54 +++++++-------- .../airbyte_ci/format/fix/java/commands.py | 6 +- .../airbyte_ci/format/fix/js/commands.py | 6 +- .../airbyte_ci/format/fix/license/commands.py | 7 +- .../airbyte_ci/format/fix/python/commands.py | 6 +- 8 files changed, 113 insertions(+), 113 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index f6aab9064eed..b157855631bf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -28,44 +28,42 @@ async def check_java(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger("format") dagger_client = ctx.params["dagger_client"] - try: - format_container = ( - dagger_client.container() - .from_(AMAZONCORRETTO_IMAGE) - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradle", - "deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), + + base_jdk_container = dagger_client.container().from_("openjdk:17.0.1-jdk-slim") + check_java_container = ( + base_jdk_container.with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_workdir("/src") - .with_exec(["./gradlew", "spotlessCheck", "--scan"]) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradle", + "deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir("/src") + ) - await format_container + try: + await check_java_container.with_exec(["./gradlew", "spotlessCheck", "--scan"]) return True - except dagger.ExecError as e: logger.error("Format check failed") logger.error(e.stderr) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index dc17fcf38bf9..85e52d93ab6a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -33,33 +33,32 @@ async def format_js(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger(f"format") dagger_client = ctx.params["dagger_client"] - try: - format_container = ( - dagger_client.container() - .from_("node:18.18.0-slim") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), + + base_node_container = dagger_client.container().from_("node:18.18.0-slim") + check_js_container = ( + base_node_container.with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash", + ] ) - .with_workdir(f"/src") - .with_exec(["npm", "install", "-g", "npm@10.1.0"]) - .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(["prettier", "--check", "."]) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_workdir("/src") + .with_exec(["npm", "install", "-g", "npm@10.1.0"]) + .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) + ) - await format_container + try: + await check_js_container.with_exec(["prettier", "--check", "."]) return True except dagger.ExecError as e: logger.error(f"Failed to format code") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index 365ca07bd37e..6e20898b8ba5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -27,26 +27,26 @@ async def format_license(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger(f"format") dagger_client = ctx.params["dagger_client"] - try: - license_container = ( - dagger_client.container() - .from_("golang:1.17") - .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "-check", "."]) + + base_go_container = dagger_client.container().from_("golang:1.17") + check_license_container = ( + base_go_container.with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) + .with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=["**/*.java", "**/*.py", "LICENSE_SHORT"], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), ) + .with_workdir("/src") + ) - await license_container + try: + await check_license_container.with_exec( + ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "--check", "."] + ) return True - except dagger.ExecError as e: logger.error("Failed to apply license") logger.error(e.stderr) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index e931e62a54dd..5880d1590116 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -32,35 +32,37 @@ async def format_python(ctx: ClickPipelineContext) -> bool: logger = logging.getLogger(f"format") dagger_client = ctx.params["dagger_client"] - try: - format_container = ( - dagger_client.container() - .from_("python:3.10.13-slim") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST - ), + + base_python_container = dagger_client.container().from_("python:3.10.13-slim") + check_python_container = ( + base_python_container.with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."]) - .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) ) + .with_mounted_directory( + "/src", + dagger_client.host().directory(".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST), + ) + .with_workdir(f"/src") + .with_exec(["poetry", "install"]) + .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."]) + .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) + ) - await format_container + try: + await ( + check_python_container.with_exec( + ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] + ).with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) + ) return True except dagger.ExecError as e: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index 185cae7b26b5..d5d8cd314921 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -19,10 +19,10 @@ async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" dagger_client = ctx.params["dagger_client"] + base_jdk_container = dagger_client.container().from_("openjdk:17.0.1-jdk-slim") + format_container = ( - dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") - .with_exec( + base_jdk_container.with_exec( sh_dash_c( [ "apt-get update", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index c0446967c413..7e99c85bd6c5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -19,10 +19,10 @@ async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" dagger_client = ctx.params["dagger_client"] + + base_node_container = dagger_client.container().from_("node:18.18.0-slim") format_container = ( - dagger_client.container() - .from_("node:18.18.0-slim") - .with_exec( + base_node_container.with_exec( sh_dash_c( [ "apt-get update", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index 80aff99a4251..e5b6df279b66 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -20,10 +20,11 @@ async def license(ctx: ClickPipelineContext): license_file = "LICENSE_SHORT" dagger_client = ctx.params["dagger_client"] + + base_go_container = dagger_client.container().from_("golang:1.17") + license_container = ( - dagger_client.container() - .from_("golang:1.17") - .with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) + base_go_container.with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) .with_mounted_directory( "/src", dagger_client.host().directory( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index 6446dea780e2..da55d0a3a4af 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -19,10 +19,10 @@ async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" dagger_client = ctx.params["dagger_client"] + base_python_container = dagger_client.container().from_("python:3.10.13-slim") + format_container = ( - dagger_client.container() - .from_("python:3.10.13-slim") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + base_python_container.with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") .with_exec( sh_dash_c( [ From 7fc97aad04d170ec8173589db328983231f59f6b Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 14:32:54 -0600 Subject: [PATCH 091/141] remove formatting stuff from connectors test package requirements --- airbyte-ci/connectors/pipelines/pipelines/consts.py | 7 ++----- .../pipelines/pipelines/dagger/containers/python.py | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 37f07eb24cc3..16581623ef7d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -13,14 +13,11 @@ CONNECTOR_TESTING_REQUIREMENTS = [ "pip==21.3.1", "mccabe==0.6.1", - "flake8==4.0.1", - "pyproject-flake8==0.0.1a2", - "black==22.3.0", - "isort==5.6.4", + # "flake8==4.0.1", + # "pyproject-flake8==0.0.1a2", "pytest==6.2.5", "coverage[toml]==6.3.1", "pytest-custom_exit_code", - "licenseheaders==0.8.8", ] BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index c9eed1bf8e85..2478f8ca377d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -61,12 +61,10 @@ def with_testing_dependencies(context: PipelineContext) -> Container: """ python_environment: Container = with_python_base(context) pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) - license_short_file = context.get_repo_dir(".", include=[LICENSE_SHORT_FILE_PATH]).file(LICENSE_SHORT_FILE_PATH) return ( python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) - .with_file(f"/{LICENSE_SHORT_FILE_PATH}", license_short_file) ) From 3126efbf0da66a429e502119fef00c0a40f3e857 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 14:46:33 -0600 Subject: [PATCH 092/141] fix imports in check/fix subcommands --- .../airbyte_ci/format/check/commands.py | 16 ++++++---------- .../pipelines/airbyte_ci/format/fix/commands.py | 14 ++++---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 7a1312ee2bcf..0b235e959f63 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -39,15 +39,11 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") if ctx.invoked_subcommand is None: - from pipelines.airbyte_ci.format.check.java.commands import java - from pipelines.airbyte_ci.format.check.js.commands import js - from pipelines.airbyte_ci.format.check.license.commands import license - from pipelines.airbyte_ci.format.check.python.commands import python - print("Running all checks...") - async with anyio.create_task_group() as check_group: - check_group.start_soon(ctx.invoke, java) - check_group.start_soon(ctx.invoke, js) - check_group.start_soon(ctx.invoke, license) - check_group.start_soon(ctx.invoke, python) + check_group.start_soon(ctx.invoke, check.get_command(ctx, "java")) + check_group.start_soon(ctx.invoke, check.get_command(ctx, "js")) + check_group.start_soon(ctx.invoke, check.get_command(ctx, "license")) + check_group.start_soon(ctx.invoke, check.get_command(ctx, "python")) + + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index bec210aff3d1..da70c0aec0c6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -30,14 +30,8 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") if ctx.invoked_subcommand is None: - from pipelines.airbyte_ci.format.fix.java.commands import java - from pipelines.airbyte_ci.format.fix.js.commands import js - from pipelines.airbyte_ci.format.fix.license.commands import license - from pipelines.airbyte_ci.format.fix.python.commands import python - print("Running all formatters...") - - await ctx.invoke(java) - await ctx.invoke(js) - await ctx.invoke(license) - await ctx.invoke(python) + await ctx.invoke(fix.get_command(ctx, "java")) + await ctx.invoke(fix.get_command(ctx, "js")) + await ctx.invoke(fix.get_command(ctx, "license")) + await ctx.invoke(fix.get_command(ctx, "python")) From ac977ca3370ae1cfd98a6847d05b8116764e13cc Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 16:28:29 -0600 Subject: [PATCH 093/141] abstract check --- .../airbyte_ci/format/check/commands.py | 2 - .../airbyte_ci/format/check/java/commands.py | 67 ++++++------------- .../airbyte_ci/format/check/js/commands.py | 53 +++------------ .../format/check/license/commands.py | 42 +++--------- .../format/check/python/commands.py | 63 ++++------------- .../airbyte_ci/format/check/utils.py | 62 +++++++++++++++++ .../pipelines/dagger/containers/python.py | 5 +- 7 files changed, 115 insertions(+), 179 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 0b235e959f63..ea40a9c76914 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -45,5 +45,3 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): check_group.start_soon(ctx.invoke, check.get_command(ctx, "js")) check_group.start_soon(ctx.invoke, check.get_command(ctx, "license")) check_group.start_soon(ctx.invoke, check.get_command(ctx, "python")) - - diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py index b157855631bf..f4c0ef8509f2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.check.utils import check from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.consts import AMAZONCORRETTO_IMAGE @@ -19,52 +20,24 @@ async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - success = await check_java(ctx) + success = await check( + ctx, + base_image="openjdk:17.0.1-jdk-slim", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + install_commands=[], + check_commands=["./gradlew spotlessCheck --scan"], + ) if not success: sys.exit(1) - - -async def check_java(ctx: ClickPipelineContext) -> bool: - logger = logging.getLogger("format") - - dagger_client = ctx.params["dagger_client"] - - base_jdk_container = dagger_client.container().from_("openjdk:17.0.1-jdk-slim") - check_java_container = ( - base_jdk_container.with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradle", - "deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir("/src") - ) - - try: - await check_java_container.with_exec(["./gradlew", "spotlessCheck", "--scan"]) - return True - except dagger.ExecError as e: - logger.error("Format check failed") - logger.error(e.stderr) - return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py index 85e52d93ab6a..671d4b027d88 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.check.utils import check from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c @@ -17,50 +18,12 @@ @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - - success = await format_js(ctx) + success = await check( + ctx, + base_image="node:18.18.0-slim", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + check_commands=["prettier --check ."], + ) if not success: sys.exit(1) - - -async def format_js(ctx: ClickPipelineContext) -> bool: - """Checks whether the repository is formatted correctly. - Args: - fix (bool): Whether to automatically fix any formatting issues detected. - Returns: - bool: True if the check/format succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - - dagger_client = ctx.params["dagger_client"] - - base_node_container = dagger_client.container().from_("node:18.18.0-slim") - check_js_container = ( - base_node_container.with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir("/src") - .with_exec(["npm", "install", "-g", "npm@10.1.0"]) - .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - ) - - try: - await check_js_container.with_exec(["prettier", "--check", "."]) - return True - except dagger.ExecError as e: - logger.error(f"Failed to format code") - logger.error(e.stderr) - return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py index 6e20898b8ba5..a1f80bda1217 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.check.utils import check from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c @@ -17,37 +18,14 @@ @click_ignore_unused_kwargs async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" - success = await format_license(ctx) + license_file = "LICENSE_SHORT" + + success = await check( + ctx, + base_image="golang:1.17", + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + check_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."], + ) if not success: sys.exit(1) - - -async def format_license(ctx: ClickPipelineContext) -> bool: - license_text = "LICENSE_SHORT" - logger = logging.getLogger(f"format") - - dagger_client = ctx.params["dagger_client"] - - base_go_container = dagger_client.container().from_("golang:1.17") - check_license_container = ( - base_go_container.with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir("/src") - ) - - try: - await check_license_container.with_exec( - ["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_text, "--check", "."] - ) - return True - except dagger.ExecError as e: - logger.error("Failed to apply license") - logger.error(e.stderr) - return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py index 5880d1590116..a8591ae0e3a9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py @@ -6,6 +6,7 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.check.utils import check from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c @@ -17,55 +18,17 @@ @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - success = await format_python(ctx) + success = await check( + ctx, + base_image="python:3.10.13-slim", + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + check_commands=[ + "poetry install", + "poetry run isort --settings-file pyproject.toml --check-only .", + "poetry run black --config pyproject.toml --check .", + ], + ) if not success: sys.exit(1) - - -async def format_python(ctx: ClickPipelineContext) -> bool: - """Checks whether the repository is formatted correctly. - Args: - fix (bool): Whether to automatically fix any formatting issues detected. - Returns: - bool: True if the check/format succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - - dagger_client = ctx.params["dagger_client"] - - base_python_container = dagger_client.container().from_("python:3.10.13-slim") - check_python_container = ( - base_python_container.with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory(".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST), - ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."]) - .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) - ) - - try: - await ( - check_python_container.with_exec( - ["poetry", "run", "isort", "--settings-file", "pyproject.toml", "--check-only", "."] - ).with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "--check", "."]) - ) - return True - - except dagger.ExecError as e: - logger.error("Format failed") - logger.error(e.stderr) - return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py new file mode 100644 index 000000000000..85ab970af4d8 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import sys +from typing import Any, Dict, List, Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +async def check( + ctx: ClickPipelineContext, + base_image: str, + include: List[str], + install_commands: List[str], + check_commands: List[str], + env_vars: Optional[Dict[str, Any]] = {}, +) -> bool: + """Checks whether the repository is formatted correctly. + Args: + ctx: (ClickPipelineContext): The pipeline context + base_image (str): The base image to use for the container + include (List[str]): The list of host files to include in the mounted directory + install_commands (List[str]): The list of commands to run to install the formatter + check_commands (List[str]): The list of commands to run to check the formatting + env_vars (Optional[Dict[str, Any]]): A dict of environment variables to set on the container + Returns: + bool: True if the check succeeded, false otherwise + """ + logger = logging.getLogger(f"format") + + dagger_client = ctx.params["dagger_client"] + + base_container = dagger_client.container().from_(base_image) + for key, value in env_vars.items(): + base_container = base_container.with_env_variable(key, value) + + check_container = ( + base_container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_exec(sh_dash_c(install_commands)) + .with_workdir("/src") + ) + + try: + await check_container.with_exec(sh_dash_c(check_commands)) + return True + except dagger.ExecError as e: + logger.error(f"Failed to format code: {e}") + return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 2478f8ca377d..45c2f8861d06 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -62,9 +62,8 @@ def with_testing_dependencies(context: PipelineContext) -> Container: python_environment: Container = with_python_base(context) pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) - return ( - python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) - .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) + return python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS).with_file( + f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file ) From ad30f84e9bc869d8766f5f3d4716c837cad73195 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 16:56:05 -0600 Subject: [PATCH 094/141] abstract fix --- .../airbyte_ci/format/check/utils.py | 47 +++++++++------- .../airbyte_ci/format/fix/java/commands.py | 56 +++++++------------ .../airbyte_ci/format/fix/js/commands.py | 36 +++--------- .../airbyte_ci/format/fix/license/commands.py | 26 +++------ .../airbyte_ci/format/fix/python/commands.py | 40 ++++--------- .../pipelines/airbyte_ci/format/fix/utils.py | 40 +++++++++++++ 6 files changed, 112 insertions(+), 133 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py index 85ab970af4d8..fb238a2fa1b0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -14,6 +14,31 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +def build_container( + ctx: ClickPipelineContext, base_image: str, include: List[str], install_commands: List[str], env_vars: Optional[Dict[str, Any]] = {} +) -> dagger.Container: + + dagger_client = ctx.params["dagger_client"] + + base_container = dagger_client.container().from_(base_image) + for key, value in env_vars.items(): + base_container = base_container.with_env_variable(key, value) + + check_container = ( + base_container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_exec(sh_dash_c(install_commands)) + .with_workdir("/src") + ) + return check_container + + async def check( ctx: ClickPipelineContext, base_image: str, @@ -35,27 +60,9 @@ async def check( """ logger = logging.getLogger(f"format") - dagger_client = ctx.params["dagger_client"] - - base_container = dagger_client.container().from_(base_image) - for key, value in env_vars.items(): - base_container = base_container.with_env_variable(key, value) - - check_container = ( - base_container.with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=include, - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_exec(sh_dash_c(install_commands)) - .with_workdir("/src") - ) - + container = build_container(ctx, base_image, include, install_commands, check_commands, env_vars) try: - await check_container.with_exec(sh_dash_c(check_commands)) + await container.with_exec(sh_dash_c(check_commands)) return True except dagger.ExecError as e: logger.error(f"Failed to format code: {e}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py index d5d8cd314921..15e0d7bbec2a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.airbyte_ci.format.fix.utils import format from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -17,41 +18,22 @@ @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - dagger_client = ctx.params["dagger_client"] - - base_jdk_container = dagger_client.container().from_("openjdk:17.0.1-jdk-slim") - - format_container = ( - base_jdk_container.with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradle", - "deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(["./gradlew", "spotlessApply"]) + await format( + ctx, + base_image="openjdk:17.0.1-jdk-slim", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + install_commands=[], + format_commands=["./gradlew spotlessApply --scan"], ) - - await format_container - await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py index 7e99c85bd6c5..5120bf283ea2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.airbyte_ci.format.fix.utils import format from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -16,33 +17,10 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): - """Format yaml and json code via prettier.""" - - dagger_client = ctx.params["dagger_client"] - - base_node_container = dagger_client.container().from_("node:18.18.0-slim") - format_container = ( - base_node_container.with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir(f"/src") - .with_exec(["npm", "install", "-g", "npm@10.1.0"]) - .with_exec(["npm", "install", "-g", "prettier@2.8.1"]) - .with_exec(["prettier", "--write", "."]) + await format( + ctx, + base_image="node:18.18.0-slim", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + format_commands=["prettier --write ."], ) - - await format_container - await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py index e5b6df279b66..381c71ca6d86 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.airbyte_ci.format.fix.utils import format from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -19,23 +20,10 @@ async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - dagger_client = ctx.params["dagger_client"] - - base_go_container = dagger_client.container().from_("golang:1.17") - - license_container = ( - base_go_container.with_exec(sh_dash_c(["apt-get update", "apt-get install -y bash tree", "go get -u github.com/google/addlicense"])) - .with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=["**/*.java", "**/*.py", "LICENSE_SHORT"], - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_workdir("/src") - .with_exec(["addlicense", "-c", "Airbyte, Inc.", "-l", "apache", "-v", "-f", license_file, "."]) + await format( + ctx, + base_image="golang:1.17", + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + format_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."], ) - - await license_container - await license_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py index da55d0a3a4af..2f6f3109c3fb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py @@ -7,6 +7,7 @@ import asyncclick as click import dagger from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.airbyte_ci.format.fix.utils import format from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -17,32 +18,15 @@ @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - dagger_client = ctx.params["dagger_client"] - - base_python_container = dagger_client.container().from_("python:3.10.13-slim") - - format_container = ( - base_python_container.with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_mounted_directory( - "/src", - dagger_client.host().directory(".", include=["**/*.py", "pyproject.toml", "poetry.lock"], exclude=DEFAULT_FORMAT_IGNORE_LIST), - ) - .with_workdir(f"/src") - .with_exec(["poetry", "install"]) - .with_exec(["poetry", "run", "isort", "--settings-file", "pyproject.toml", "."]) - .with_exec(["poetry", "run", "black", "--config", "pyproject.toml", "."]) + await format( + ctx, + base_image="python:3.10.13-slim", + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + format_commands=[ + "poetry install", + "poetry run isort --settings-file pyproject.toml .", + "poetry run black --config pyproject.toml .", + ], ) - - await format_container - await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py new file mode 100644 index 000000000000..8deac6415c25 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py @@ -0,0 +1,40 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import sys +from typing import Any, Dict, List, Optional + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.check.utils import build_container +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + + +async def format( + ctx: ClickPipelineContext, + base_image: str, + include: List[str], + install_commands: List[str], + format_commands: List[str], + env_vars: Optional[Dict[str, Any]] = {}, +) -> bool: + """Formats the repository. + Args: + ctx: (ClickPipelineContext): The pipeline context + base_image (str): The base image to use for the container + include (List[str]): The list of host files to include in the mounted directory + install_commands (List[str]): The list of commands to run to install the formatter + format_commands (List[str]): The list of commands to run to check the formatting + env_vars (Optional[Dict[str, Any]]): A dict of environment variables to set on the container + Returns: + bool: True if the check succeeded, false otherwise + """ + container = build_container(ctx, base_image, include, install_commands, env_vars) + + format_container = container.with_exec(sh_dash_c(format_commands)) + await format_container.directory("/src").export(".") From 7b126dd7555d752b4a559a1ae9e889ed8cb8e7cd Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 17:01:55 -0600 Subject: [PATCH 095/141] merge fix --- .../airbyte_ci/format/fix/commands.py | 90 ++++++++++++++++--- .../airbyte_ci/format/fix/java/commands.py | 39 -------- .../airbyte_ci/format/fix/js/commands.py | 26 ------ .../airbyte_ci/format/fix/license/commands.py | 29 ------ .../airbyte_ci/format/fix/python/commands.py | 32 ------- 5 files changed, 79 insertions(+), 137 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index da70c0aec0c6..be36ce1ea6d9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -4,20 +4,14 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.fix.utils import format from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.group( - cls=LazyGroup, help="Run code format checks and fix any failures.", - lazy_subcommands={ - "java": "pipelines.airbyte_ci.format.fix.java.commands.java", - "js": "pipelines.airbyte_ci.format.fix.js.commands.js", - "license": "pipelines.airbyte_ci.format.fix.license.commands.license", - "python": "pipelines.airbyte_ci.format.fix.python.commands.python", - }, invoke_without_command=True, chain=True, ) @@ -31,7 +25,81 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): if ctx.invoked_subcommand is None: print("Running all formatters...") - await ctx.invoke(fix.get_command(ctx, "java")) - await ctx.invoke(fix.get_command(ctx, "js")) - await ctx.invoke(fix.get_command(ctx, "license")) - await ctx.invoke(fix.get_command(ctx, "python")) + await ctx.invoke(java) + await ctx.invoke(js) + await ctx.invoke(license) + await ctx.invoke(python) + + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): + """Format java, groovy, and sql code via spotless.""" + await format( + ctx, + base_image="openjdk:17.0.1-jdk-slim", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + install_commands=[], + format_commands=["./gradlew spotlessApply --scan"], + ) + + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + await format( + ctx, + base_image="node:18.18.0-slim", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + format_commands=["prettier --write ."], + ) + + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Add license to python and java code via addlicense.""" + license_file = "LICENSE_SHORT" + + await format( + ctx, + base_image="golang:1.17", + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + format_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."], + ) + + +@fix.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Format python code via black and isort.""" + await format( + ctx, + base_image="python:3.10.13-slim", + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + format_commands=[ + "poetry install", + "poetry run isort --settings-file pyproject.toml .", + "poetry run black --config pyproject.toml .", + ], + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py deleted file mode 100644 index 15e0d7bbec2a..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/java/commands.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.airbyte_ci.format.fix.utils import format -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): - """Format java, groovy, and sql code via spotless.""" - await format( - ctx, - base_image="openjdk:17.0.1-jdk-slim", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - install_commands=[], - format_commands=["./gradlew spotlessApply --scan"], - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py deleted file mode 100644 index 5120bf283ea2..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/js/commands.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.airbyte_ci.format.fix.utils import format -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - await format( - ctx, - base_image="node:18.18.0-slim", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - format_commands=["prettier --write ."], - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py deleted file mode 100644 index 381c71ca6d86..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/license/commands.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.airbyte_ci.format.fix.utils import format -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Add license to python and java code via addlicense.""" - license_file = "LICENSE_SHORT" - - await format( - ctx, - base_image="golang:1.17", - include=["**/*.java", "**/*.py", license_file], - install_commands=["go get -u github.com/google/addlicense"], - format_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."], - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py deleted file mode 100644 index 2f6f3109c3fb..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/python/commands.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.airbyte_ci.format.fix.utils import format -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Format python code via black and isort.""" - await format( - ctx, - base_image="python:3.10.13-slim", - env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, - include=["**/*.py", "pyproject.toml", "poetry.lock"], - install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - format_commands=[ - "poetry install", - "poetry run isort --settings-file pyproject.toml .", - "poetry run black --config pyproject.toml .", - ], - ) From 9f4467d9320214fcbcd9ddeca37aed4618425327 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 17:39:10 -0600 Subject: [PATCH 096/141] merge check --- .../airbyte_ci/format/check/commands.py | 94 ++++++++++++++++--- .../airbyte_ci/format/check/java/commands.py | 43 --------- .../airbyte_ci/format/check/js/commands.py | 29 ------ .../format/check/license/commands.py | 31 ------ .../format/check/python/commands.py | 34 ------- .../airbyte_ci/format/check/utils.py | 11 +-- .../airbyte_ci/format/fix/commands.py | 10 +- .../pipelines/airbyte_ci/format/fix/utils.py | 2 +- 8 files changed, 91 insertions(+), 163 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index ea40a9c76914..9395c77a558b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,11 +1,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import sys from typing import Optional import anyio import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.java.commands import java +from pipelines.airbyte_ci.format.check.utils import run_check from pipelines.cli.click_decorators import ( LazyPassDecorator, click_append_to_context_object, @@ -19,14 +20,7 @@ @click.group( - cls=LazyGroup, help="Run code format checks and fail if any checks fail.", - lazy_subcommands={ - "java": "pipelines.airbyte_ci.format.check.java.commands.java", - "js": "pipelines.airbyte_ci.format.check.js.commands.js", - "license": "pipelines.airbyte_ci.format.check.license.commands.license", - "python": "pipelines.airbyte_ci.format.check.python.commands.python", - }, invoke_without_command=True, chain=True, ) @@ -41,7 +35,83 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): if ctx.invoked_subcommand is None: print("Running all checks...") async with anyio.create_task_group() as check_group: - check_group.start_soon(ctx.invoke, check.get_command(ctx, "java")) - check_group.start_soon(ctx.invoke, check.get_command(ctx, "js")) - check_group.start_soon(ctx.invoke, check.get_command(ctx, "license")) - check_group.start_soon(ctx.invoke, check.get_command(ctx, "python")) + check_group.start_soon(ctx.invoke, java) + check_group.start_soon(ctx.invoke, js) + check_group.start_soon(ctx.invoke, license) + check_group.start_soon(ctx.invoke, python) + + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext): + """Format java, groovy, and sql code via spotless.""" + + await run_check( + ctx, + base_image="openjdk:17.0.1-jdk-slim", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + install_commands=[], + check_commands=["./gradlew spotlessCheck --scan"], + ) + + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext): + """Format yaml and json code via prettier.""" + await run_check( + ctx, + base_image="node:18.18.0-slim", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + check_commands=["prettier --check ."], + ) + + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license(ctx: ClickPipelineContext): + """Add license to python and java code via addlicense.""" + license_file = "LICENSE_SHORT" + + await run_check( + ctx, + base_image="golang:1.17", + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + check_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."], + ) + + +@check.command() +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext): + """Format python code via black and isort.""" + await run_check( + ctx, + base_image="python:3.10.13-slim", + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + check_commands=[ + "poetry install", + "poetry run isort --settings-file pyproject.toml --check-only .", + "poetry run black --config pyproject.toml --check .", + ], + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py deleted file mode 100644 index f4c0ef8509f2..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/java/commands.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.check.utils import check -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.consts import AMAZONCORRETTO_IMAGE -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): - """Format java, groovy, and sql code via spotless.""" - - success = await check( - ctx, - base_image="openjdk:17.0.1-jdk-slim", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - install_commands=[], - check_commands=["./gradlew spotlessCheck --scan"], - ) - if not success: - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py deleted file mode 100644 index 671d4b027d88..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/js/commands.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.check.utils import check -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): - """Format yaml and json code via prettier.""" - success = await check( - ctx, - base_image="node:18.18.0-slim", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - check_commands=["prettier --check ."], - ) - if not success: - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py deleted file mode 100644 index a1f80bda1217..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/license/commands.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.check.utils import check -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): - """Add license to python and java code via addlicense.""" - license_file = "LICENSE_SHORT" - - success = await check( - ctx, - base_image="golang:1.17", - include=["**/*.java", "**/*.py", license_file], - install_commands=["go get -u github.com/google/addlicense"], - check_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."], - ) - if not success: - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py deleted file mode 100644 index a8591ae0e3a9..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/python/commands.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.check.utils import check -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context - - -@click.command() -@pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): - """Format python code via black and isort.""" - success = await check( - ctx, - base_image="python:3.10.13-slim", - env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, - include=["**/*.py", "pyproject.toml", "poetry.lock"], - install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - check_commands=[ - "poetry install", - "poetry run isort --settings-file pyproject.toml --check-only .", - "poetry run black --config pyproject.toml --check .", - ], - ) - if not success: - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py index fb238a2fa1b0..f6ef0f5660a9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -39,7 +39,7 @@ def build_container( return check_container -async def check( +async def run_check( ctx: ClickPipelineContext, base_image: str, include: List[str], @@ -60,10 +60,5 @@ async def check( """ logger = logging.getLogger(f"format") - container = build_container(ctx, base_image, include, install_commands, check_commands, env_vars) - try: - await container.with_exec(sh_dash_c(check_commands)) - return True - except dagger.ExecError as e: - logger.error(f"Failed to format code: {e}") - return False + container = build_container(ctx, base_image, include, install_commands, env_vars) + await container.with_exec(sh_dash_c(check_commands)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index be36ce1ea6d9..b7c475311d41 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -4,7 +4,7 @@ import asyncclick as click import dagger -from pipelines.airbyte_ci.format.fix.utils import format +from pipelines.airbyte_ci.format.fix.utils import run_format from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -36,7 +36,7 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - await format( + await run_format( ctx, base_image="openjdk:17.0.1-jdk-slim", include=[ @@ -61,7 +61,7 @@ async def java(ctx: ClickPipelineContext): @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): - await format( + await run_format( ctx, base_image="node:18.18.0-slim", include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], @@ -77,7 +77,7 @@ async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - await format( + await run_format( ctx, base_image="golang:1.17", include=["**/*.java", "**/*.py", license_file], @@ -91,7 +91,7 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - await format( + await run_format( ctx, base_image="python:3.10.13-slim", env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py index 8deac6415c25..aea360df1237 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py @@ -15,7 +15,7 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) -async def format( +async def run_format( ctx: ClickPipelineContext, base_image: str, include: List[str], From 435e55bf5a2a2fac0f1671ab2a34e538bea119b9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:08:09 -0600 Subject: [PATCH 097/141] split container building from running check/format --- .../airbyte_ci/format/check/commands.py | 32 ++++++++++--------- .../airbyte_ci/format/check/utils.py | 14 +++++++- .../airbyte_ci/format/fix/commands.py | 31 ++++++++++-------- .../pipelines/airbyte_ci/format/fix/utils.py | 20 +++--------- 4 files changed, 52 insertions(+), 45 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 9395c77a558b..be14d27762c5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -6,7 +6,7 @@ import anyio import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.utils import run_check +from pipelines.airbyte_ci.format.check.utils import build_container, run_check, run_check_old from pipelines.cli.click_decorators import ( LazyPassDecorator, click_append_to_context_object, @@ -46,8 +46,7 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - - await run_check( + container = build_container( ctx, base_image="openjdk:17.0.1-jdk-slim", include=[ @@ -64,8 +63,9 @@ async def java(ctx: ClickPipelineContext): "tools/gradle/codestyle/sql-dbeaver.properties", ], install_commands=[], - check_commands=["./gradlew spotlessCheck --scan"], ) + check_commands = ["./gradlew spotlessCheck --scan"] + await run_check(container, check_commands) @check.command() @@ -73,13 +73,14 @@ async def java(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - await run_check( + container = build_container( ctx, base_image="node:18.18.0-slim", include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - check_commands=["prettier --check ."], ) + check_commands = ["prettier --check ."] + await run_check(container, check_commands) @check.command() @@ -88,14 +89,14 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - - await run_check( + container = build_container( ctx, base_image="golang:1.17", include=["**/*.java", "**/*.py", license_file], install_commands=["go get -u github.com/google/addlicense"], - check_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."], ) + check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] + await run_check(container, check_commands) @check.command() @@ -103,15 +104,16 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - await run_check( + container = build_container( ctx, base_image="python:3.10.13-slim", env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, include=["**/*.py", "pyproject.toml", "poetry.lock"], install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - check_commands=[ - "poetry install", - "poetry run isort --settings-file pyproject.toml --check-only .", - "poetry run black --config pyproject.toml --check .", - ], ) + check_commands = [ + "poetry install", + "poetry run isort --settings-file pyproject.toml --check-only .", + "poetry run black --config pyproject.toml --check .", + ] + await run_check(container, check_commands) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py index f6ef0f5660a9..fe54ab565a82 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -39,7 +39,7 @@ def build_container( return check_container -async def run_check( +async def run_check_old( ctx: ClickPipelineContext, base_image: str, include: List[str], @@ -62,3 +62,15 @@ async def run_check( container = build_container(ctx, base_image, include, install_commands, env_vars) await container.with_exec(sh_dash_c(check_commands)) + + +async def run_check( + container: dagger.Container, + check_commands: List[str], +) -> dagger.Container: + """Checks whether the repository is formatted correctly. + Args: + container: (dagger.Container): The container to run the formatting check in + check_commands (List[str]): The list of commands to run to check the formatting + """ + await container.with_exec(sh_dash_c(check_commands)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index b7c475311d41..5cd98a54dd91 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -4,9 +4,11 @@ import asyncclick as click import dagger +from pipelines.airbyte_ci.format.check.utils import build_container from pipelines.airbyte_ci.format.fix.utils import run_format from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup +from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -36,7 +38,7 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - await run_format( + container = build_container( ctx, base_image="openjdk:17.0.1-jdk-slim", include=[ @@ -53,21 +55,23 @@ async def java(ctx: ClickPipelineContext): "tools/gradle/codestyle/sql-dbeaver.properties", ], install_commands=[], - format_commands=["./gradlew spotlessApply --scan"], ) + format_commands = ["./gradlew spotlessApply --scan"] + await run_format(container, format_commands) @fix.command() @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): - await run_format( + container = build_container( ctx, base_image="node:18.18.0-slim", include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - format_commands=["prettier --write ."], ) + format_commands = ["prettier --write ."] + await run_format(container, format_commands) @fix.command() @@ -76,14 +80,14 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - - await run_format( + container = build_container( ctx, base_image="golang:1.17", include=["**/*.java", "**/*.py", license_file], install_commands=["go get -u github.com/google/addlicense"], - format_commands=[f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."], ) + format_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."] + await run_format(container, format_commands) @fix.command() @@ -91,15 +95,16 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - await run_format( + container = build_container( ctx, base_image="python:3.10.13-slim", env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, include=["**/*.py", "pyproject.toml", "poetry.lock"], install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - format_commands=[ - "poetry install", - "poetry run isort --settings-file pyproject.toml .", - "poetry run black --config pyproject.toml .", - ], ) + format_commands = [ + "poetry install", + "poetry run isort --settings-file pyproject.toml .", + "poetry run black --config pyproject.toml .", + ] + await run_format(container, format_commands) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py index aea360df1237..a4044f76894c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py @@ -16,25 +16,13 @@ async def run_format( - ctx: ClickPipelineContext, - base_image: str, - include: List[str], - install_commands: List[str], + container: dagger.Container, format_commands: List[str], - env_vars: Optional[Dict[str, Any]] = {}, -) -> bool: +) -> dagger.Container: """Formats the repository. Args: - ctx: (ClickPipelineContext): The pipeline context - base_image (str): The base image to use for the container - include (List[str]): The list of host files to include in the mounted directory - install_commands (List[str]): The list of commands to run to install the formatter - format_commands (List[str]): The list of commands to run to check the formatting - env_vars (Optional[Dict[str, Any]]): A dict of environment variables to set on the container - Returns: - bool: True if the check succeeded, false otherwise + container: (dagger.Container): The container to run the formatter in + format_commands (List[str]): The list of commands to run to format the repository """ - container = build_container(ctx, base_image, include, install_commands, env_vars) - format_container = container.with_exec(sh_dash_c(format_commands)) await format_container.directory("/src").export(".") From 2ea4c44db47e358f275a61246725fdf7f684185f Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:18:33 -0600 Subject: [PATCH 098/141] shared containers --- .../airbyte_ci/format/check/commands.py | 49 +++-------- .../airbyte_ci/format/check/utils.py | 50 ----------- .../pipelines/airbyte_ci/format/containers.py | 86 +++++++++++++++++++ .../airbyte_ci/format/fix/commands.py | 48 +++-------- .../pipelines/airbyte_ci/format/fix/utils.py | 1 - 5 files changed, 107 insertions(+), 127 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index be14d27762c5..e9eb5b1f45b6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -6,7 +6,13 @@ import anyio import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.utils import build_container, run_check, run_check_old +from pipelines.airbyte_ci.format.check.utils import run_check +from pipelines.airbyte_ci.format.containers import ( + format_java_container, + format_js_container, + format_license_container, + format_python_container, +) from pipelines.cli.click_decorators import ( LazyPassDecorator, click_append_to_context_object, @@ -46,24 +52,7 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - container = build_container( - ctx, - base_image="openjdk:17.0.1-jdk-slim", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - install_commands=[], - ) + container = format_java_container(ctx) check_commands = ["./gradlew spotlessCheck --scan"] await run_check(container, check_commands) @@ -73,12 +62,7 @@ async def java(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - container = build_container( - ctx, - base_image="node:18.18.0-slim", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - ) + container = format_js_container(ctx) check_commands = ["prettier --check ."] await run_check(container, check_commands) @@ -89,12 +73,7 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - container = build_container( - ctx, - base_image="golang:1.17", - include=["**/*.java", "**/*.py", license_file], - install_commands=["go get -u github.com/google/addlicense"], - ) + container = format_license_container(ctx, license_file) check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] await run_check(container, check_commands) @@ -104,13 +83,7 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - container = build_container( - ctx, - base_image="python:3.10.13-slim", - env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, - include=["**/*.py", "pyproject.toml", "poetry.lock"], - install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - ) + container = format_python_container(ctx) check_commands = [ "poetry install", "poetry run isort --settings-file pyproject.toml --check-only .", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py index fe54ab565a82..6bd71167cd2e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -14,56 +14,6 @@ pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) -def build_container( - ctx: ClickPipelineContext, base_image: str, include: List[str], install_commands: List[str], env_vars: Optional[Dict[str, Any]] = {} -) -> dagger.Container: - - dagger_client = ctx.params["dagger_client"] - - base_container = dagger_client.container().from_(base_image) - for key, value in env_vars.items(): - base_container = base_container.with_env_variable(key, value) - - check_container = ( - base_container.with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=include, - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_exec(sh_dash_c(install_commands)) - .with_workdir("/src") - ) - return check_container - - -async def run_check_old( - ctx: ClickPipelineContext, - base_image: str, - include: List[str], - install_commands: List[str], - check_commands: List[str], - env_vars: Optional[Dict[str, Any]] = {}, -) -> bool: - """Checks whether the repository is formatted correctly. - Args: - ctx: (ClickPipelineContext): The pipeline context - base_image (str): The base image to use for the container - include (List[str]): The list of host files to include in the mounted directory - install_commands (List[str]): The list of commands to run to install the formatter - check_commands (List[str]): The list of commands to run to check the formatting - env_vars (Optional[Dict[str, Any]]): A dict of environment variables to set on the container - Returns: - bool: True if the check succeeded, false otherwise - """ - logger = logging.getLogger(f"format") - - container = build_container(ctx, base_image, include, install_commands, env_vars) - await container.with_exec(sh_dash_c(check_commands)) - - async def run_check( container: dagger.Container, check_commands: List[str], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py new file mode 100644 index 000000000000..1c140805eb23 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List, Optional + +import click +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + + +def build_container( + ctx: ClickPipelineContext, base_image: str, include: List[str], install_commands: List[str], env_vars: Optional[Dict[str, Any]] = {} +) -> dagger.Container: + + dagger_client = ctx.params["dagger_client"] + + base_container = dagger_client.container().from_(base_image) + for key, value in env_vars.items(): + base_container = base_container.with_env_variable(key, value) + + check_container = ( + base_container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + .with_exec(sh_dash_c(install_commands)) + .with_workdir("/src") + ) + return check_container + + +def format_java_container(ctx: click.Context) -> dagger.Container: + """Format java, groovy, and sql code via spotless.""" + return build_container( + ctx, + base_image="openjdk:17.0.1-jdk-slim", + include=[ + "**/*.java", + "**/*.sql", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + "tools/gradle/codestyle/sql-dbeaver.properties", + ], + install_commands=[], + ) + + +def format_js_container(ctx: click.Context) -> dagger.Container: + """Format yaml and json code via prettier.""" + return build_container( + ctx, + base_image="node:18.18.0-slim", + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + ) + + +def format_license_container(ctx: click.Context, license_file: str) -> dagger.Container: + return build_container( + ctx, + base_image="golang:1.17", + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + ) + + +def format_python_container(ctx: click.Context) -> dagger.Container: + """Format python code via black and isort.""" + return build_container( + ctx, + base_image="python:3.10.13-slim", + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 5cd98a54dd91..7168cdc9c1b5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -4,7 +4,12 @@ import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.utils import build_container +from pipelines.airbyte_ci.format.containers import ( + format_java_container, + format_js_container, + format_license_container, + format_python_container, +) from pipelines.airbyte_ci.format.fix.utils import run_format from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup @@ -38,24 +43,7 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - container = build_container( - ctx, - base_image="openjdk:17.0.1-jdk-slim", - include=[ - "**/*.java", - "**/*.sql", - "**/*.gradle", - "gradlew", - "gradlew.bat", - "gradle", - "**/deps.toml", - "**/gradle.properties", - "**/version.properties", - "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", - ], - install_commands=[], - ) + container = format_java_container(ctx) format_commands = ["./gradlew spotlessApply --scan"] await run_format(container, format_commands) @@ -64,12 +52,7 @@ async def java(ctx: ClickPipelineContext): @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): - container = build_container( - ctx, - base_image="node:18.18.0-slim", - include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], - ) + container = format_js_container(ctx) format_commands = ["prettier --write ."] await run_format(container, format_commands) @@ -80,12 +63,7 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - container = build_container( - ctx, - base_image="golang:1.17", - include=["**/*.java", "**/*.py", license_file], - install_commands=["go get -u github.com/google/addlicense"], - ) + container = format_license_container(ctx, license_file) format_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."] await run_format(container, format_commands) @@ -95,13 +73,7 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - container = build_container( - ctx, - base_image="python:3.10.13-slim", - env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, - include=["**/*.py", "pyproject.toml", "poetry.lock"], - install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], - ) + container = format_python_container(ctx) format_commands = [ "poetry install", "poetry run isort --settings-file pyproject.toml .", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py index a4044f76894c..bf9af673ee42 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py @@ -6,7 +6,6 @@ import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.utils import build_container from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c From aa5ced4b289225485ad497f5b1ab69cf08986c1c Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:28:25 -0600 Subject: [PATCH 099/141] shared actions --- .../format/{fix/utils.py => actions.py} | 20 ++++++----- .../airbyte_ci/format/check/commands.py | 18 +++------- .../airbyte_ci/format/check/utils.py | 26 -------------- .../airbyte_ci/format/fix/commands.py | 4 +-- .../pipelines/dagger/containers/go.py | 35 +++++++++++++++++++ 5 files changed, 54 insertions(+), 49 deletions(-) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/{fix/utils.py => actions.py} (56%) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py similarity index 56% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index bf9af673ee42..41d39076e4a7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -1,17 +1,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import logging -import sys -from typing import Any, Dict, List, Optional +from typing import List -import asyncclick as click import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + +async def run_check( + container: dagger.Container, + check_commands: List[str], +) -> dagger.Container: + """Checks whether the repository is formatted correctly. + Args: + container: (dagger.Container): The container to run the formatting check in + check_commands (List[str]): The list of commands to run to check the formatting + """ + await container.with_exec(sh_dash_c(check_commands)) async def run_format( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index e9eb5b1f45b6..f01565b84d18 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,28 +1,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import sys -from typing import Optional +from typing import List import anyio import asyncclick as click import dagger -from pipelines.airbyte_ci.format.check.utils import run_check +from pipelines.airbyte_ci.format.actions import run_check from pipelines.airbyte_ci.format.containers import ( format_java_container, format_js_container, format_license_container, format_python_container, ) -from pipelines.cli.click_decorators import ( - LazyPassDecorator, - click_append_to_context_object, - click_ignore_unused_kwargs, - click_merge_args_into_context_obj, -) -from pipelines.cli.lazy_group import LazyGroup -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.group( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py deleted file mode 100644 index 6bd71167cd2e..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -import logging -import sys -from typing import Any, Dict, List, Optional - -import asyncclick as click -import dagger -from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs -from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - - -async def run_check( - container: dagger.Container, - check_commands: List[str], -) -> dagger.Container: - """Checks whether the repository is formatted correctly. - Args: - container: (dagger.Container): The container to run the formatting check in - check_commands (List[str]): The list of commands to run to check the formatting - """ - await container.with_exec(sh_dash_c(check_commands)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 7168cdc9c1b5..aa6b20c517de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,16 +1,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Optional +from typing import List, Optional import asyncclick as click import dagger +from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( format_java_container, format_js_container, format_license_container, format_python_container, ) -from pipelines.airbyte_ci.format.fix.utils import run_format from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.utils import sh_dash_c diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py new file mode 100644 index 000000000000..eedd18711c6c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +def with_go(context: PipelineContext, go_version: str = "3.10") -> Container: + """Build a Python container with a cache volume for pip cache. + + Args: + context (PipelineContext): The current test context, providing a dagger client and a repository directory. + python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim". + + Raises: + ValueError: Raised if the python_image_name is not a python image. + + Returns: + Container: The python base environment container. + """ + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + + base_container = ( + context.dagger_client.container() + .from_(f"python:{python_version}-slim") + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 git", + "pip install pip==23.1.2", + ] + ) + ) + ) + + return base_container From 769b792a463589ed33a9875a92ea5bd96b000884 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:30:41 -0600 Subject: [PATCH 100/141] remove unused imports --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 -- .../pipelines/pipelines/airbyte_ci/format/commands.py | 3 --- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 5 ----- 3 files changed, 10 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index f01565b84d18..0ff3606df52b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -4,7 +4,6 @@ import anyio import asyncclick as click -import dagger from pipelines.airbyte_ci.format.actions import run_check from pipelines.airbyte_ci.format.containers import ( format_java_container, @@ -13,7 +12,6 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 2ef9ca80e7bf..c325ef2a89d2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -6,10 +6,7 @@ Module exposing the format command. """ -from typing import Optional - import asyncclick as click -import dagger from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index aa6b20c517de..dc71dd66e816 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,9 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import List, Optional - import asyncclick as click -import dagger from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( format_java_container, @@ -12,8 +9,6 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.cli.lazy_group import LazyGroup -from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context From 8774791a03da3f28f2c49f484058a27580ccf599 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:41:01 -0600 Subject: [PATCH 101/141] split up and document container building: --- .../pipelines/airbyte_ci/format/containers.py | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 1c140805eb23..56eda9c910dc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -10,28 +10,50 @@ def build_container( - ctx: ClickPipelineContext, base_image: str, include: List[str], install_commands: List[str], env_vars: Optional[Dict[str, Any]] = {} + ctx: ClickPipelineContext, + base_image: str, + include: List[str], + install_commands: Optional[List[str]], + env_vars: Optional[Dict[str, Any]] = {}, ) -> dagger.Container: - + """Build a container for formatting code. + Args: + ctx (ClickPipelineContext): The context of the pipeline + base_image (str): The base image to use for the container + include (List[str]): The list of files to include in the container + install_commands (Optional[List[str]]): The list of commands to run to install dependencies for the formatter + env_vars (Optional[Dict[str, Any]]): The list of environment variables to set on the container + Returns: + dagger.Container: The container to use for formatting + """ dagger_client = ctx.params["dagger_client"] - base_container = dagger_client.container().from_(base_image) + # Create container from base image + container = dagger_client.container().from_(base_image) + + # Add any environment variables for key, value in env_vars.items(): - base_container = base_container.with_env_variable(key, value) - - check_container = ( - base_container.with_mounted_directory( - "/src", - dagger_client.host().directory( - ".", - include=include, - exclude=DEFAULT_FORMAT_IGNORE_LIST, - ), - ) - .with_exec(sh_dash_c(install_commands)) - .with_workdir("/src") + container = container.with_env_variable(key, value) + + # Mount the relevant parts of the repository: the code to format and the formatting config + # Exclude the default ignore list to keep things as small as possible + container = container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), ) - return check_container + + # Install any dependencies of the formatter + if install_commands: + container = container.with_exec(sh_dash_c(install_commands)) + + # Set the working directory to the code to format + container = container.with_workdir("/src") + + return container def format_java_container(ctx: click.Context) -> dagger.Container: @@ -52,7 +74,6 @@ def format_java_container(ctx: click.Context) -> dagger.Container: "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties", ], - install_commands=[], ) From 31514ed9dca1648f6d19e98d306d9b98471fa90a Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:48:13 -0600 Subject: [PATCH 102/141] install dependencies before mounting code. add skip_entrypoint=true --- .../pipelines/pipelines/airbyte_ci/format/actions.py | 4 ++-- .../pipelines/airbyte_ci/format/containers.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 41d39076e4a7..7c3fb0bbd5fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -15,7 +15,7 @@ async def run_check( container: (dagger.Container): The container to run the formatting check in check_commands (List[str]): The list of commands to run to check the formatting """ - await container.with_exec(sh_dash_c(check_commands)) + await container.with_exec(sh_dash_c(check_commands), skip_entrypoint=True) async def run_format( @@ -27,5 +27,5 @@ async def run_format( container: (dagger.Container): The container to run the formatter in format_commands (List[str]): The list of commands to run to format the repository """ - format_container = container.with_exec(sh_dash_c(format_commands)) + format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) await format_container.directory("/src").export(".") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 56eda9c910dc..01be02fc0521 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -13,7 +13,7 @@ def build_container( ctx: ClickPipelineContext, base_image: str, include: List[str], - install_commands: Optional[List[str]], + install_commands: Optional[List[str]] = None, env_vars: Optional[Dict[str, Any]] = {}, ) -> dagger.Container: """Build a container for formatting code. @@ -35,6 +35,10 @@ def build_container( for key, value in env_vars.items(): container = container.with_env_variable(key, value) + # Install any dependencies of the formatter + if install_commands: + container = container.with_exec(sh_dash_c(install_commands), skip_entrypoint=True) + # Mount the relevant parts of the repository: the code to format and the formatting config # Exclude the default ignore list to keep things as small as possible container = container.with_mounted_directory( @@ -46,10 +50,6 @@ def build_container( ), ) - # Install any dependencies of the formatter - if install_commands: - container = container.with_exec(sh_dash_c(install_commands)) - # Set the working directory to the code to format container = container.with_workdir("/src") From 60505e9a69bb81a06a85dc1809d17909c5ebf105 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 18:53:19 -0600 Subject: [PATCH 103/141] rename pipelines --- .../connectors/pipelines/pipelines/airbyte_ci/format/actions.py | 1 - .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 +- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 7c3fb0bbd5fe..55d9c1ab896a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -5,7 +5,6 @@ import dagger from pipelines.helpers.utils import sh_dash_c - async def run_check( container: dagger.Container, check_commands: List[str], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 0ff3606df52b..65f914361547 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -26,7 +26,7 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fail if any checks fail.""" # TODO: fix this client hacking - ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") + ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Check repository formatting") if ctx.invoked_subcommand is None: print("Running all checks...") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index dc71dd66e816..5e1cfb351753 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -23,7 +23,7 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fix any failures.""" # TODO: fix this client hacking - ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format License") + ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format repository") if ctx.invoked_subcommand is None: print("Running all formatters...") From ba6ce1a892938bb4db8e22124ac0ca83849b3967 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 19:05:48 -0600 Subject: [PATCH 104/141] register all tasks without listing them, poetry install no root --- .../pipelines/pipelines/airbyte_ci/format/actions.py | 1 + .../pipelines/airbyte_ci/format/check/commands.py | 8 +++----- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 8 +++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 55d9c1ab896a..7c3fb0bbd5fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -5,6 +5,7 @@ import dagger from pipelines.helpers.utils import sh_dash_c + async def run_check( container: dagger.Container, check_commands: List[str], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 65f914361547..46b9e06bd266 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -31,10 +31,8 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): if ctx.invoked_subcommand is None: print("Running all checks...") async with anyio.create_task_group() as check_group: - check_group.start_soon(ctx.invoke, java) - check_group.start_soon(ctx.invoke, js) - check_group.start_soon(ctx.invoke, license) - check_group.start_soon(ctx.invoke, python) + for command in check.commands.values(): + check_group.start_soon(ctx.invoke, command) @check.command() @@ -75,7 +73,7 @@ async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" container = format_python_container(ctx) check_commands = [ - "poetry install", + "poetry install --no-root", "poetry run isort --settings-file pyproject.toml --check-only .", "poetry run black --config pyproject.toml --check .", ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 5e1cfb351753..6790cb0cfe99 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -27,10 +27,8 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): if ctx.invoked_subcommand is None: print("Running all formatters...") - await ctx.invoke(java) - await ctx.invoke(js) - await ctx.invoke(license) - await ctx.invoke(python) + for command in fix.commands.values(): + await ctx.invoke(command) @fix.command() @@ -70,7 +68,7 @@ async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" container = format_python_container(ctx) format_commands = [ - "poetry install", + "poetry install --no-root", "poetry run isort --settings-file pyproject.toml .", "poetry run black --config pyproject.toml .", ] From 8a8c2ecd81b5d8790e3ea80fc89ad3c2c7473d05 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 19:32:31 -0600 Subject: [PATCH 105/141] some cleanup/add an example of a direction we could go in --- .../pipelines/airbyte_ci/format/actions.py | 20 +++++++++++ .../pipelines/airbyte_ci/format/containers.py | 20 ++++++++--- .../pipelines/dagger/containers/go.py | 35 ------------------- .../pipelines/dagger/containers/python.py | 1 - 4 files changed, 36 insertions(+), 40 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 7c3fb0bbd5fe..465b70133ed9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -29,3 +29,23 @@ async def run_format( """ format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) await format_container.directory("/src").export(".") + + +def mount_repo_for_formatting( + container: dagger.Container, + include: List[str], +) -> dagger.Container: + """Mounts the relevant parts of the repository: the code to format and the formatting config + Args: + container: (dagger.Container): The container to mount the repository in + include (List[str]): The list of files to include in the container + """ + container = container.with_mounted_directory( + "/src", + dagger.host().directory( + ".", + include=include, + ), + ).with_workdir("/src") + + return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 01be02fc0521..4cb21c8d709d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -4,7 +4,10 @@ import click import dagger +from pipelines.airbyte_ci.format.actions import mount_repo_for_formatting from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.dagger.actions.python.pipx import with_installed_pipx_package, with_pipx +from pipelines.dagger.containers.python import with_python_base from pipelines.helpers.utils import sh_dash_c from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext @@ -56,7 +59,7 @@ def build_container( return container -def format_java_container(ctx: click.Context) -> dagger.Container: +def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: """Format java, groovy, and sql code via spotless.""" return build_container( ctx, @@ -77,7 +80,7 @@ def format_java_container(ctx: click.Context) -> dagger.Container: ) -def format_js_container(ctx: click.Context) -> dagger.Container: +def format_js_container(ctx: ClickPipelineContext) -> dagger.Container: """Format yaml and json code via prettier.""" return build_container( ctx, @@ -87,7 +90,7 @@ def format_js_container(ctx: click.Context) -> dagger.Container: ) -def format_license_container(ctx: click.Context, license_file: str) -> dagger.Container: +def format_license_container(ctx: ClickPipelineContext, license_file: str) -> dagger.Container: return build_container( ctx, base_image="golang:1.17", @@ -96,8 +99,17 @@ def format_license_container(ctx: click.Context, license_file: str) -> dagger.Co ) -def format_python_container(ctx: click.Context) -> dagger.Container: +def format_python_container(ctx: ClickPipelineContext) -> dagger.Container: """Format python code via black and isort.""" + + # Here's approximately what it would look like if we built it the other way. Doesn't + # quite work. Not sure if its putting more work into. + + # base_container = with_python_base(ctx._click_context, python_version="3.10.13") + # container = with_installed_pipx_package(ctx._click_context, base_container, "poetry") + # container = mount_repo_for_formatting(container, include=["**/*.py", "pyproject.toml", "poetry.lock"]) + # return container + return build_container( ctx, base_image="python:3.10.13-slim", diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py deleted file mode 100644 index eedd18711c6c..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/go.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - - -def with_go(context: PipelineContext, go_version: str = "3.10") -> Container: - """Build a Python container with a cache volume for pip cache. - - Args: - context (PipelineContext): The current test context, providing a dagger client and a repository directory. - python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim". - - Raises: - ValueError: Raised if the python_image_name is not a python image. - - Returns: - Container: The python base environment container. - """ - - pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - - base_container = ( - context.dagger_client.container() - .from_(f"python:{python_version}-slim") - .with_mounted_cache("/root/.cache/pip", pip_cache) - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 git", - "pip install pip==23.1.2", - ] - ) - ) - ) - - return base_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index 45c2f8861d06..98227fe9c81d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -6,7 +6,6 @@ from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import ( CONNECTOR_TESTING_REQUIREMENTS, - LICENSE_SHORT_FILE_PATH, PIP_CACHE_PATH, PIP_CACHE_VOLUME_NAME, POETRY_CACHE_PATH, From 29343bc60dcfb9f91fe1df993aea6eab78e761b6 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 19:57:38 -0600 Subject: [PATCH 106/141] little things --- .../pipelines/pipelines/airbyte_ci/format/actions.py | 3 ++- .../connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 465b70133ed9..690d531b38ae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -32,6 +32,7 @@ async def run_format( def mount_repo_for_formatting( + dagger_client: dagger.Client, container: dagger.Container, include: List[str], ) -> dagger.Container: @@ -42,7 +43,7 @@ def mount_repo_for_formatting( """ container = container.with_mounted_directory( "/src", - dagger.host().directory( + dagger_client.host().directory( ".", include=include, ), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 5221636dd281..982da85e00f2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -69,7 +69,6 @@ async def _run(self) -> StepResult: "gradle.properties", "gradle", "gradlew", - "LICENSE_SHORT", "settings.gradle", "build.gradle", "tools/gradle", From bfc44fcb3d1e4534d3218143599e06825972efd2 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 21:16:49 -0600 Subject: [PATCH 107/141] put go and node images in constants --- .../pipelines/pipelines/airbyte_ci/format/containers.py | 7 ++++--- airbyte-ci/connectors/pipelines/pipelines/consts.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 4cb21c8d709d..8112ae8bbd36 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -6,6 +6,7 @@ import dagger from pipelines.airbyte_ci.format.actions import mount_repo_for_formatting from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.consts import GO_IMAGE, JDK_IMAGE, NODE_IMAGE from pipelines.dagger.actions.python.pipx import with_installed_pipx_package, with_pipx from pipelines.dagger.containers.python import with_python_base from pipelines.helpers.utils import sh_dash_c @@ -63,7 +64,7 @@ def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: """Format java, groovy, and sql code via spotless.""" return build_container( ctx, - base_image="openjdk:17.0.1-jdk-slim", + base_image=JDK_IMAGE, include=[ "**/*.java", "**/*.sql", @@ -84,7 +85,7 @@ def format_js_container(ctx: ClickPipelineContext) -> dagger.Container: """Format yaml and json code via prettier.""" return build_container( ctx, - base_image="node:18.18.0-slim", + base_image=NODE_IMAGE, include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], ) @@ -93,7 +94,7 @@ def format_js_container(ctx: ClickPipelineContext) -> dagger.Container: def format_license_container(ctx: ClickPipelineContext, license_file: str) -> dagger.Container: return build_container( ctx, - base_image="golang:1.17", + base_image=GO_IMAGE, include=["**/*.java", "**/*.py", license_file], install_commands=["go get -u github.com/google/addlicense"], ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 16581623ef7d..d4d529e60d9a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -30,6 +30,9 @@ } LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()] AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" +NODE_IMAGE = "node:18.18.0-slim" +JDK_IMAGE = "openjdk:17.0.1-jdk-slim" +GO_IMAGE = "golang:1.17" DOCKER_VERSION = "24.0.2" DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli" From 90cc182387a29a6e3f74b5027f077d1361a94991 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 21:23:55 -0600 Subject: [PATCH 108/141] use amazoncoretto image for java --- .../pipelines/pipelines/airbyte_ci/format/containers.py | 9 +++++++-- airbyte-ci/connectors/pipelines/pipelines/consts.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 8112ae8bbd36..3120e9ba5e3b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -6,7 +6,7 @@ import dagger from pipelines.airbyte_ci.format.actions import mount_repo_for_formatting from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.consts import GO_IMAGE, JDK_IMAGE, NODE_IMAGE +from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, JDK_IMAGE, NODE_IMAGE from pipelines.dagger.actions.python.pipx import with_installed_pipx_package, with_pipx from pipelines.dagger.containers.python import with_python_base from pipelines.helpers.utils import sh_dash_c @@ -64,7 +64,7 @@ def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: """Format java, groovy, and sql code via spotless.""" return build_container( ctx, - base_image=JDK_IMAGE, + base_image=AMAZONCORRETTO_IMAGE, include=[ "**/*.java", "**/*.sql", @@ -78,6 +78,11 @@ def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: "tools/gradle/codestyle/java-google-style.xml", "tools/gradle/codestyle/sql-dbeaver.properties", ], + install_commands=[ + "yum update -y", + "yum install -y findutils", # gradle requires xargs, which is shipped in findutils. + "yum clean all", + ] ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index d4d529e60d9a..af12d41b34ca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -31,7 +31,6 @@ LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()] AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" NODE_IMAGE = "node:18.18.0-slim" -JDK_IMAGE = "openjdk:17.0.1-jdk-slim" GO_IMAGE = "golang:1.17" DOCKER_VERSION = "24.0.2" DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" From 3d57d94dd0c366604f2fb4f265c9897932cd69d2 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Sun, 5 Nov 2023 22:57:01 -0600 Subject: [PATCH 109/141] log check output in a friendly way and exit 1 on errors --- .../airbyte_ci/format/check/commands.py | 30 ++++++++++-- .../airbyte_ci/format/check/utils.py | 49 +++++++++++++++++++ .../pipelines/airbyte_ci/format/containers.py | 4 +- .../airbyte_ci/format/fix/commands.py | 5 +- 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 46b9e06bd266..dedac5b343de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,10 +1,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import logging +import sys from typing import List import anyio import asyncclick as click +import dagger from pipelines.airbyte_ci.format.actions import run_check +from pipelines.airbyte_ci.format.check.utils import log_output from pipelines.airbyte_ci.format.containers import ( format_java_container, format_js_container, @@ -20,19 +24,35 @@ invoke_without_command=True, chain=True, ) +@click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext): +async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext, list_errors: bool): """Run code format checks and fail if any checks fail.""" - # TODO: fix this client hacking + logger = logging.getLogger("check") + ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Check repository formatting") + ctx.obj["check_results"] = {} if ctx.invoked_subcommand is None: - print("Running all checks...") + logger.info("Running all checks...") async with anyio.create_task_group() as check_group: - for command in check.commands.values(): - check_group.start_soon(ctx.invoke, command) + for command in ctx.command.commands.values(): + check_group.start_soon(run_check_command, ctx, command) + + log_output(ctx.obj["check_results"], list_errors, logger) + + if any(not succeeded for (succeeded, _) in ctx.obj["check_results"].values()): + sys.exit(1) + + +async def run_check_command(ctx, command): + try: + await ctx.invoke(command) + ctx.obj["check_results"][command.name] = (True, None) + except dagger.ExecError as e: + ctx.obj["check_results"][command.name] = (False, str(e)) @check.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py new file mode 100644 index 000000000000..561b2fbb95dd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py @@ -0,0 +1,49 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Dict, Tuple + +from jinja2 import Template + +SUMMARY_TEMPLATE_STR = """ + +Summary of Check Results +======================== +{% for command_name, result in check_results.items() %} +{{ '✅' if result[0] else '❌' }} {{ command_prefix }} {{ command_name }} +{% endfor %} +""" + +DETAILS_TEMPLATE_STR = """ + +Detailed Errors for Failed Checks +================================= +{% for command_name, error in failed_commands_details %} +❌ {{ command_prefix }} {{ command_name }} + +Error: {{ error }} +{% endfor %} +================================= + +""" + + +def log_output(check_results: Dict[str, Tuple[bool, str]], list_errors, logger): + command_prefix = "airbyte-ci check" + + summary_template = Template(SUMMARY_TEMPLATE_STR) + summary_message = summary_template.render(check_results=check_results, command_prefix=command_prefix) + logger.info(summary_message) + + result_contains_failures = any(not succeeded for (succeeded, _) in check_results.values()) + + if result_contains_failures: + if list_errors: + failed_commands_details = [(name, error) for name, (success, error) in check_results.items() if not success] + if failed_commands_details: + details_template = Template(DETAILS_TEMPLATE_STR) + details_message = details_template.render(failed_commands_details=failed_commands_details, command_prefix=command_prefix) + logger.info(details_message) + else: + logger.info("Run `airbyte-ci format check --list-errors` to see detailed error messages for failed checks.") + + logger.info("Run `airbyte-ci format fix` to fix formatting errors.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 3120e9ba5e3b..0a8787117014 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -6,7 +6,7 @@ import dagger from pipelines.airbyte_ci.format.actions import mount_repo_for_formatting from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, JDK_IMAGE, NODE_IMAGE +from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, NODE_IMAGE from pipelines.dagger.actions.python.pipx import with_installed_pipx_package, with_pipx from pipelines.dagger.containers.python import with_python_base from pipelines.helpers.utils import sh_dash_c @@ -82,7 +82,7 @@ def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: "yum update -y", "yum install -y findutils", # gradle requires xargs, which is shipped in findutils. "yum clean all", - ] + ], ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 6790cb0cfe99..502a3dbfcdf0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,5 +1,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import logging import asyncclick as click from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( @@ -22,11 +23,11 @@ @click_ignore_unused_kwargs async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fix any failures.""" - # TODO: fix this client hacking + logger = logging.getLogger("format") ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format repository") if ctx.invoked_subcommand is None: - print("Running all formatters...") + logger.info("Running all formatters...") for command in fix.commands.values(): await ctx.invoke(command) From ca9a66895187ab22574c3ac9e0b3c19cb677e812 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 6 Nov 2023 01:05:57 -0600 Subject: [PATCH 110/141] list errors in ci --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e5047ebfdf73..aff00c51e9ad 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -51,7 +51,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format check" + subcommand: "format check --list-errors" notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" From c9c11ca1b6b7b323dbe015acb043d6c650f921c9 Mon Sep 17 00:00:00 2001 From: erohmensing Date: Mon, 6 Nov 2023 01:07:03 -0600 Subject: [PATCH 111/141] format --- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 502a3dbfcdf0..e5b8a33e4dbb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,6 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import logging + import asyncclick as click from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( From be9bb2ecc31b80c2288cedc149f6d77456847d6e Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 7 Nov 2023 08:54:23 -0700 Subject: [PATCH 112/141] Remove dagger client in the context obj --- .../airbyte_ci/format/check/commands.py | 13 +++++++----- .../pipelines/airbyte_ci/format/containers.py | 20 +++++++++---------- .../airbyte_ci/format/fix/commands.py | 13 +++++++----- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index dedac5b343de..200a6aacbbfd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -32,7 +32,6 @@ async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext, list_err """Run code format checks and fail if any checks fail.""" logger = logging.getLogger("check") - ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Check repository formatting") ctx.obj["check_results"] = {} if ctx.invoked_subcommand is None: @@ -60,7 +59,8 @@ async def run_check_command(ctx, command): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - container = format_java_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Check java formatting") + container = format_java_container(dagger_client) check_commands = ["./gradlew spotlessCheck --scan"] await run_check(container, check_commands) @@ -70,7 +70,8 @@ async def java(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): """Format yaml and json code via prettier.""" - container = format_js_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Check js formatting") + container = format_js_container(dagger_client) check_commands = ["prettier --check ."] await run_check(container, check_commands) @@ -81,7 +82,8 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - container = format_license_container(ctx, license_file) + dagger_client = await ctx.get_dagger_client(pipeline_name="Check license header") + container = format_license_container(dagger_client, license_file) check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] await run_check(container, check_commands) @@ -91,7 +93,8 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - container = format_python_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Check python formatting") + container = format_python_container(dagger_client) check_commands = [ "poetry install --no-root", "poetry run isort --settings-file pyproject.toml --check-only .", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 0a8787117014..71a047c47df9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -14,7 +14,7 @@ def build_container( - ctx: ClickPipelineContext, + dagger_client: dagger.Client, base_image: str, include: List[str], install_commands: Optional[List[str]] = None, @@ -30,8 +30,6 @@ def build_container( Returns: dagger.Container: The container to use for formatting """ - dagger_client = ctx.params["dagger_client"] - # Create container from base image container = dagger_client.container().from_(base_image) @@ -60,10 +58,10 @@ def build_container( return container -def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: +def format_java_container(dagger_client: dagger.Client) -> dagger.Container: """Format java, groovy, and sql code via spotless.""" return build_container( - ctx, + dagger_client, base_image=AMAZONCORRETTO_IMAGE, include=[ "**/*.java", @@ -86,26 +84,26 @@ def format_java_container(ctx: ClickPipelineContext) -> dagger.Container: ) -def format_js_container(ctx: ClickPipelineContext) -> dagger.Container: +def format_js_container(dagger_client: dagger.Client) -> dagger.Container: """Format yaml and json code via prettier.""" return build_container( - ctx, + dagger_client, base_image=NODE_IMAGE, include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], ) -def format_license_container(ctx: ClickPipelineContext, license_file: str) -> dagger.Container: +def format_license_container(dagger_client: dagger.Client, license_file: str) -> dagger.Container: return build_container( - ctx, + dagger_client, base_image=GO_IMAGE, include=["**/*.java", "**/*.py", license_file], install_commands=["go get -u github.com/google/addlicense"], ) -def format_python_container(ctx: ClickPipelineContext) -> dagger.Container: +def format_python_container(dagger_client: dagger.Client) -> dagger.Container: """Format python code via black and isort.""" # Here's approximately what it would look like if we built it the other way. Doesn't @@ -117,7 +115,7 @@ def format_python_container(ctx: ClickPipelineContext) -> dagger.Container: # return container return build_container( - ctx, + dagger_client, base_image="python:3.10.13-slim", env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, include=["**/*.py", "pyproject.toml", "poetry.lock"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index e5b8a33e4dbb..20a85d40fe88 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -25,7 +25,6 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): """Run code format checks and fix any failures.""" logger = logging.getLogger("format") - ctx.obj["dagger_client"] = await pipeline_ctx.get_dagger_client(pipeline_name="Format repository") if ctx.invoked_subcommand is None: logger.info("Running all formatters...") @@ -38,7 +37,8 @@ async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - container = format_java_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Format java") + container = format_java_container(dagger_client) format_commands = ["./gradlew spotlessApply --scan"] await run_format(container, format_commands) @@ -47,7 +47,8 @@ async def java(ctx: ClickPipelineContext): @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext): - container = format_js_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Format js") + container = format_js_container(dagger_client) format_commands = ["prettier --write ."] await run_format(container, format_commands) @@ -58,7 +59,8 @@ async def js(ctx: ClickPipelineContext): async def license(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - container = format_license_container(ctx, license_file) + dagger_client = await ctx.get_dagger_client(pipeline_name="Add license") + container = format_license_container(dagger_client, license_file) format_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."] await run_format(container, format_commands) @@ -68,7 +70,8 @@ async def license(ctx: ClickPipelineContext): @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext): """Format python code via black and isort.""" - container = format_python_container(ctx) + dagger_client = await ctx.get_dagger_client(pipeline_name="Format python") + container = format_python_container(dagger_client) format_commands = [ "poetry install --no-root", "poetry run isort --settings-file pyproject.toml .", From afe5cea3cd1efe3ab041bfb7e7437813d14d4628 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 7 Nov 2023 09:43:22 -0700 Subject: [PATCH 113/141] Make run_sub_commands_generic --- .../airbyte_ci/format/check/commands.py | 34 ++----- .../airbyte_ci/format/check/utils.py | 49 --------- .../airbyte_ci/format/fix/commands.py | 7 +- .../pipelines/pipelines/helpers/cli.py | 99 +++++++++++++++++++ 4 files changed, 109 insertions(+), 80 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 200a6aacbbfd..d2d91cf0ed18 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,14 +1,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import logging -import sys from typing import List -import anyio import asyncclick as click -import dagger from pipelines.airbyte_ci.format.actions import run_check -from pipelines.airbyte_ci.format.check.utils import log_output from pipelines.airbyte_ci.format.containers import ( format_java_container, format_js_container, @@ -16,6 +11,7 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.cli import LogOptions, run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -26,32 +22,16 @@ ) @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") @click_merge_args_into_context_obj -@pass_pipeline_context @click_ignore_unused_kwargs -async def check(ctx: click.Context, pipeline_ctx: ClickPipelineContext, list_errors: bool): +async def check(ctx: click.Context, list_errors: bool): """Run code format checks and fail if any checks fail.""" - logger = logging.getLogger("check") - - ctx.obj["check_results"] = {} if ctx.invoked_subcommand is None: - logger.info("Running all checks...") - async with anyio.create_task_group() as check_group: - for command in ctx.command.commands.values(): - check_group.start_soon(run_check_command, ctx, command) - - log_output(ctx.obj["check_results"], list_errors, logger) - - if any(not succeeded for (succeeded, _) in ctx.obj["check_results"].values()): - sys.exit(1) - - -async def run_check_command(ctx, command): - try: - await ctx.invoke(command) - ctx.obj["check_results"][command.name] = (True, None) - except dagger.ExecError as e: - ctx.obj["check_results"][command.name] = (False, str(e)) + log_options = LogOptions( + list_errors=list_errors, + help_message="Run `airbyte-ci format check --list-errors` to see detailed error messages for failed checks.", + ) + await run_all_subcommands(ctx, log_options) @check.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py deleted file mode 100644 index 561b2fbb95dd..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/utils.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from typing import Dict, Tuple - -from jinja2 import Template - -SUMMARY_TEMPLATE_STR = """ - -Summary of Check Results -======================== -{% for command_name, result in check_results.items() %} -{{ '✅' if result[0] else '❌' }} {{ command_prefix }} {{ command_name }} -{% endfor %} -""" - -DETAILS_TEMPLATE_STR = """ - -Detailed Errors for Failed Checks -================================= -{% for command_name, error in failed_commands_details %} -❌ {{ command_prefix }} {{ command_name }} - -Error: {{ error }} -{% endfor %} -================================= - -""" - - -def log_output(check_results: Dict[str, Tuple[bool, str]], list_errors, logger): - command_prefix = "airbyte-ci check" - - summary_template = Template(SUMMARY_TEMPLATE_STR) - summary_message = summary_template.render(check_results=check_results, command_prefix=command_prefix) - logger.info(summary_message) - - result_contains_failures = any(not succeeded for (succeeded, _) in check_results.values()) - - if result_contains_failures: - if list_errors: - failed_commands_details = [(name, error) for name, (success, error) in check_results.items() if not success] - if failed_commands_details: - details_template = Template(DETAILS_TEMPLATE_STR) - details_message = details_template.render(failed_commands_details=failed_commands_details, command_prefix=command_prefix) - logger.info(details_message) - else: - logger.info("Run `airbyte-ci format check --list-errors` to see detailed error messages for failed checks.") - - logger.info("Run `airbyte-ci format fix` to fix formatting errors.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 20a85d40fe88..3e24477cbffc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -11,6 +11,7 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.cli import run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -20,16 +21,14 @@ chain=True, ) @click_merge_args_into_context_obj -@pass_pipeline_context @click_ignore_unused_kwargs -async def fix(ctx: click.Context, pipeline_ctx: ClickPipelineContext): +async def fix(ctx: click.Context): """Run code format checks and fix any failures.""" logger = logging.getLogger("format") if ctx.invoked_subcommand is None: logger.info("Running all formatters...") - for command in fix.commands.values(): - await ctx.invoke(command) + await run_all_subcommands(ctx) @fix.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py new file mode 100644 index 000000000000..65f7532f0649 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -0,0 +1,99 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import sys +from dataclasses import dataclass +from typing import Dict, Tuple + +import anyio +import asyncclick as click +import dagger +from jinja2 import Template + +ALL_RESULTS_KEY = "_run_all_results" + +SUMMARY_TEMPLATE_STR = """ + +Summary of Sub Command Results +======================== +{% for command_name, result in results.items() %} +{{ '✅' if result[0] else '❌' }} {{ command_prefix }} {{ command_name }} +{% endfor %} +""" + +DETAILS_TEMPLATE_STR = """ + +Detailed Errors for Failed Sub Commands +================================= +{% for command_name, error in failed_commands_details %} +❌ {{ command_prefix }} {{ command_name }} + +Error: {{ error }} +{% endfor %} +================================= + +""" + + +@dataclass +class LogOptions: + list_errors: bool = False + help_message: str = None + + +def _log_output(ctx: click.Context, logger, options: LogOptions = LogOptions()): + """ + Log the output of the subcommands run by `run_all_subcommands`. + """ + subcommand_results = ctx.obj[ALL_RESULTS_KEY] + command_path = ctx.command_path + + summary_template = Template(SUMMARY_TEMPLATE_STR) + summary_message = summary_template.render(results=subcommand_results, command_prefix=command_path) + logger.info(summary_message) + + result_contains_failures = any(not succeeded for (succeeded, _) in subcommand_results.values()) + + if result_contains_failures: + if options.list_errors: + failed_commands_details = [(name, error) for name, (success, error) in subcommand_results.items() if not success] + if failed_commands_details: + details_template = Template(DETAILS_TEMPLATE_STR) + details_message = details_template.render(failed_commands_details=failed_commands_details, command_prefix=command_path) + logger.info(details_message) + else: + logger.info(f"Run `{command_path} --list-errors` to see detailed error messages for failed checks.") + + if options.help_message: + logger.info(options.help_message) + + +async def _run_sub_command(ctx: click.Context, command: click.Command): + """ + Run a subcommand and store the result in the context object. + """ + try: + await ctx.invoke(command) + ctx.obj[ALL_RESULTS_KEY][command.name] = (True, None) + except dagger.ExecError as e: + ctx.obj[ALL_RESULTS_KEY][command.name] = (False, str(e)) + + +async def run_all_subcommands(ctx: click.Context, log_options: LogOptions = LogOptions()): + """ + Run all subcommands of a given command and log the results. + """ + ctx.obj[ALL_RESULTS_KEY] = {} + command_path = ctx.command_path + command_name = ctx.command.name + logger = logging.getLogger(command_name) + + logger.info(f"Running all sub commands of {command_path}...") + async with anyio.create_task_group() as check_group: + for command in ctx.command.commands.values(): + check_group.start_soon(_run_sub_command, ctx, command) + + _log_output(ctx, logger, log_options) + + if any(not succeeded for (succeeded, _) in ctx.obj[ALL_RESULTS_KEY].values()): + sys.exit(1) From 22b19f37d3880618ed897c0afb686482ccf54f16 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 7 Nov 2023 18:16:53 -0700 Subject: [PATCH 114/141] Add all command --- airbyte-ci/connectors/pipelines/README.md | 14 +++++----- .../airbyte_ci/format/check/commands.py | 27 ++++++++++--------- .../airbyte_ci/format/fix/commands.py | 21 +++++++-------- .../pipelines/pipelines/helpers/cli.py | 17 ++++++++---- 4 files changed, 43 insertions(+), 36 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index b4795ba58a64..27588421837a 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -374,24 +374,24 @@ Migrate source-openweather to use the base image: `airbyte-ci connectors --name= ### `format` command subgroup Available commands: -* `airbyte-ci format check` -* `airbyte-ci format fix` +* `airbyte-ci format check all` +* `airbyte-ci format fix all` ### Examples -- Check for formatting errors in the repository: `airbyte-ci format check` +- Check for formatting errors in the repository: `airbyte-ci format check all` - Fix formatting for only python files: `airbyte-ci format fix python` -### `format check` command +### `format check all` command -This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix`. +This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix all`. Running `airbyte-ci format check` will run checks on all different types of code. Run `airbyte-ci format check --help` for subcommands to check formatting for only certain types of files. -### `format fix` command +### `format fix all` command This command runs formatting checks and reformats any code that would be reformatted, so it's recommended to stage changes you might have before running this command. -Running `airbyte-ci format fix` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. +Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. ### `metadata` command subgroup diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index d2d91cf0ed18..c8bc401c6d6b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -17,21 +17,24 @@ @click.group( help="Run code format checks and fail if any checks fail.", - invoke_without_command=True, chain=True, ) -@click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") -@click_merge_args_into_context_obj -@click_ignore_unused_kwargs -async def check(ctx: click.Context, list_errors: bool): - """Run code format checks and fail if any checks fail.""" +async def check(): + pass + - if ctx.invoked_subcommand is None: - log_options = LogOptions( - list_errors=list_errors, - help_message="Run `airbyte-ci format check --list-errors` to see detailed error messages for failed checks.", - ) - await run_all_subcommands(ctx, log_options) +@check.command() +@click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") +@click.pass_context +async def all(ctx: click.Context, list_errors: bool): + """ + Run all format checks and fail if any checks fail. + """ + log_options = LogOptions( + list_errors=list_errors, + help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks.", + ) + await run_all_subcommands(ctx, log_options) @check.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 3e24477cbffc..0e619450c14b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,7 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import logging - import asyncclick as click from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( @@ -10,25 +8,24 @@ format_license_container, format_python_container, ) -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.cli import run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.group( help="Run code format checks and fix any failures.", - invoke_without_command=True, chain=True, ) -@click_merge_args_into_context_obj -@click_ignore_unused_kwargs -async def fix(ctx: click.Context): - """Run code format checks and fix any failures.""" - logger = logging.getLogger("format") +async def fix(): + pass - if ctx.invoked_subcommand is None: - logger.info("Running all formatters...") - await run_all_subcommands(ctx) + +@fix.command() +@click.pass_context +async def all(ctx: click.Context): + """Run code format checks and fix any failures.""" + await run_all_subcommands(ctx) @fix.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index 65f7532f0649..c78a243a39c0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -84,13 +84,20 @@ async def run_all_subcommands(ctx: click.Context, log_options: LogOptions = LogO Run all subcommands of a given command and log the results. """ ctx.obj[ALL_RESULTS_KEY] = {} - command_path = ctx.command_path - command_name = ctx.command.name - logger = logging.getLogger(command_name) - logger.info(f"Running all sub commands of {command_path}...") + parent_command_path = ctx.parent.command_path + parent_command_name = ctx.parent.command.name + current_command_name = ctx.command.name + + logger = logging.getLogger(parent_command_name) + + # omit current command from list of subcommands + all_subcommands_dict = ctx.parent.command.commands + filtered_subcommands_dict = {name: command for name, command in all_subcommands_dict.items() if name != current_command_name} + + logger.info(f"Running all sub commands of {parent_command_path}...") async with anyio.create_task_group() as check_group: - for command in ctx.command.commands.values(): + for command in filtered_subcommands_dict.values(): check_group.start_soon(_run_sub_command, ctx, command) _log_output(ctx, logger, log_options) From a9cb03d19711dfe8d6371ad92e30975c5c92a488 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 8 Nov 2023 12:04:27 +0100 Subject: [PATCH 115/141] update format command signature --- .../pipelines/pipelines/airbyte_ci/format/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index c325ef2a89d2..992c4187c7a2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -23,5 +23,5 @@ @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def format(ctx: click.Context, pipeline_ctx: ClickPipelineContext): +async def format(pipeline_context: ClickPipelineContext): pass From 8327acda409da82ae96dc7b25b925eb46e2a2b1b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 8 Nov 2023 12:07:56 +0100 Subject: [PATCH 116/141] avoid pyton reserved function name in command name --- .../pipelines/airbyte_ci/connectors/commands.py | 2 +- .../pipelines/airbyte_ci/connectors/list/commands.py | 4 ++-- .../pipelines/airbyte_ci/format/check/commands.py | 9 +++++---- .../pipelines/pipelines/airbyte_ci/format/commands.py | 3 ++- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 539c839f94fb..cf25122268bc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -145,7 +145,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: lazy_subcommands={ "build": "pipelines.airbyte_ci.connectors.build_image.commands.build", "test": "pipelines.airbyte_ci.connectors.test.commands.test", - "list": "pipelines.airbyte_ci.connectors.list.commands.list", + "list": "pipelines.airbyte_ci.connectors.list.commands.list_connectors", "publish": "pipelines.airbyte_ci.connectors.publish.commands.publish", "bump_version": "pipelines.airbyte_ci.connectors.bump_version.commands.bump_version", "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 4b0f54e6c462..76c50648eb2e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -9,9 +9,9 @@ from rich.text import Text -@click.command(cls=DaggerPipelineCommand, help="List all selected connectors.") +@click.command(cls=DaggerPipelineCommand, help="List all selected connectors.", name="list") @click.pass_context -async def list( +async def list_connectors( ctx: click.Context, ): selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index c8bc401c6d6b..894d3322b84c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -1,6 +1,7 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from typing import List import asyncclick as click from pipelines.airbyte_ci.format.actions import run_check @@ -10,7 +11,7 @@ format_license_container, format_python_container, ) -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.helpers.cli import LogOptions, run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -23,10 +24,10 @@ async def check(): pass -@check.command() +@check.command(name="all") @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") @click.pass_context -async def all(ctx: click.Context, list_errors: bool): +async def all_languages(ctx: click.Context, list_errors: bool): """ Run all format checks and fail if any checks fail. """ diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py index 992c4187c7a2..efa340d91c47 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -14,6 +14,7 @@ @click.group( cls=LazyGroup, + name="format", help="Commands related to formatting.", lazy_subcommands={ "check": "pipelines.airbyte_ci.format.check.commands.check", @@ -23,5 +24,5 @@ @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def format(pipeline_context: ClickPipelineContext): +async def format_code(pipeline_context: ClickPipelineContext): pass diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 7bf5bef2f946..3a8912b986c8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -226,7 +226,7 @@ async def get_modified_files_str(ctx: click.Context): help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", - "format": "pipelines.airbyte_ci.format.commands.format", + "format": "pipelines.airbyte_ci.format.commands.format_code", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", }, From de813e5e0199f436bca9b01e731f2aa161b24202 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 8 Nov 2023 12:15:57 +0100 Subject: [PATCH 117/141] check commands: decorator cleanup, all_languages uses pipeline_context --- .../airbyte_ci/format/check/commands.py | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 894d3322b84c..df15ac8097a1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -11,7 +11,7 @@ format_license_container, format_python_container, ) -from pipelines.cli.click_decorators import click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.cli import LogOptions, run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -26,24 +26,25 @@ async def check(): @check.command(name="all") @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") -@click.pass_context -async def all_languages(ctx: click.Context, list_errors: bool): +@pass_pipeline_context +@click_merge_args_into_context_obj +@click_ignore_unused_kwargs +async def all_languages(pipeline_context: ClickPipelineContext): """ Run all format checks and fail if any checks fail. """ log_options = LogOptions( - list_errors=list_errors, + list_errors=pipeline_context.params["list_errors"], help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks.", ) - await run_all_subcommands(ctx, log_options) + await run_all_subcommands(pipeline_context, log_options) @check.command() @pass_pipeline_context -@click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): +async def java(pipeline_context: ClickPipelineContext): """Format java, groovy, and sql code via spotless.""" - dagger_client = await ctx.get_dagger_client(pipeline_name="Check java formatting") + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check java formatting") container = format_java_container(dagger_client) check_commands = ["./gradlew spotlessCheck --scan"] await run_check(container, check_commands) @@ -51,10 +52,9 @@ async def java(ctx: ClickPipelineContext): @check.command() @pass_pipeline_context -@click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): +async def js(pipeline_context: ClickPipelineContext): """Format yaml and json code via prettier.""" - dagger_client = await ctx.get_dagger_client(pipeline_name="Check js formatting") + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check js formatting") container = format_js_container(dagger_client) check_commands = ["prettier --check ."] await run_check(container, check_commands) @@ -62,11 +62,10 @@ async def js(ctx: ClickPipelineContext): @check.command() @pass_pipeline_context -@click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): +async def license(pipeline_context: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" - dagger_client = await ctx.get_dagger_client(pipeline_name="Check license header") + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check license header") container = format_license_container(dagger_client, license_file) check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] await run_check(container, check_commands) @@ -74,10 +73,9 @@ async def license(ctx: ClickPipelineContext): @check.command() @pass_pipeline_context -@click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): +async def python(pipeline_context: ClickPipelineContext): """Format python code via black and isort.""" - dagger_client = await ctx.get_dagger_client(pipeline_name="Check python formatting") + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check python formatting") container = format_python_container(dagger_client) check_commands = [ "poetry install --no-root", From 67de99dcfcdf87ac97b65bcf1f22b876c44902c6 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 8 Nov 2023 15:37:47 +0100 Subject: [PATCH 118/141] fix decorators, pipeline_context can't be used at the all level --- .../pipelines/airbyte_ci/format/check/commands.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index df15ac8097a1..c288b19b7beb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -11,7 +11,6 @@ format_license_container, format_python_container, ) -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.helpers.cli import LogOptions, run_all_subcommands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @@ -26,18 +25,16 @@ async def check(): @check.command(name="all") @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") -@pass_pipeline_context -@click_merge_args_into_context_obj -@click_ignore_unused_kwargs -async def all_languages(pipeline_context: ClickPipelineContext): +@click.pass_context +async def all_languages(ctx: click.Context, list_errors: bool): """ Run all format checks and fail if any checks fail. """ log_options = LogOptions( - list_errors=pipeline_context.params["list_errors"], + list_errors=list_errors, help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks.", ) - await run_all_subcommands(pipeline_context, log_options) + await run_all_subcommands(ctx, log_options) @check.command() From b05f0b7127165d7a63af88b165e96ff56784e8c4 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 8 Nov 2023 18:08:45 -0700 Subject: [PATCH 119/141] Instantiate early --- .../pipelines/airbyte_ci/format/check/commands.py | 4 +++- .../connectors/pipelines/pipelines/models/singleton.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index c288b19b7beb..33ca7eec7c56 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -26,10 +26,12 @@ async def check(): @check.command(name="all") @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") @click.pass_context -async def all_languages(ctx: click.Context, list_errors: bool): +@pass_pipeline_context +async def all_languages(ctx: click.Context, pipeline_context: ClickPipelineContext, list_errors: bool): """ Run all format checks and fail if any checks fail. """ + await pipeline_context.get_dagger_client(pipeline_name="Check all languages") log_options = LogOptions( list_errors=list_errors, help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index e9d759d401f6..6204e84f10e7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,7 +1,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. from typing import Any, Type - +import threading class Singleton: """ @@ -16,9 +16,12 @@ class Singleton: _instances: dict[Type["Singleton"], Any] = {} _initialized: dict[Type["Singleton"], bool] = {} + _lock = threading.Lock() + def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: - cls._instances[cls] = super().__new__(cls) - cls._initialized[cls] = False + with cls._lock: + cls._instances[cls] = super().__new__(cls) + cls._initialized[cls] = False return cls._instances[cls] From dff2d49c8a948098b714e47efd0ef33136e302d0 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 8 Nov 2023 18:13:10 -0700 Subject: [PATCH 120/141] Update fix command --- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 6 ++++-- .../connectors/pipelines/pipelines/models/singleton.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index 0e619450c14b..f101cc28c4c1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -21,10 +21,12 @@ async def fix(): pass -@fix.command() +@fix.command(name="all") @click.pass_context -async def all(ctx: click.Context): +@pass_pipeline_context +async def all_languages(ctx: click.Context, pipeline_context: ClickPipelineContext): """Run code format checks and fix any failures.""" + await pipeline_context.get_dagger_client(pipeline_name="Fix all languages") await run_all_subcommands(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 6204e84f10e7..2d884dc136ed 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,7 +1,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. -from typing import Any, Type import threading +from typing import Any, Type + class Singleton: """ @@ -18,7 +19,6 @@ class Singleton: _initialized: dict[Type["Singleton"], bool] = {} _lock = threading.Lock() - def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: with cls._lock: From 7772cf3a37c09a4fee17c5143e1502bafa56c5dd Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 13:48:18 +0100 Subject: [PATCH 121/141] update format.yaml to run format check all --list-errors --- .github/workflows/format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index aff00c51e9ad..de9cc6a1dec5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -51,7 +51,7 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format check --list-errors" + subcommand: "format check all --list-errors" notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" From 3f2ab69c91cb2c96f46bc18a41e97baf41aca870 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 13:49:25 +0100 Subject: [PATCH 122/141] update format.yml to cancel-in-progress --- .github/workflows/format.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index de9cc6a1dec5..fa90a300b2a2 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -2,6 +2,8 @@ name: Check for formatting errors concurrency: group: ${{ github.workflow }}-${{ github.ref }} + # Cancel any previous runs on the same branch if they are still in progress + cancel-in-progress: true on: workflow_dispatch: From 34cb674a2b049fbfee74370d000e605c49f03c9b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 13:56:00 +0100 Subject: [PATCH 123/141] improve help message to suggest fix --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 33ca7eec7c56..c18a006dc130 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -34,7 +34,7 @@ async def all_languages(ctx: click.Context, pipeline_context: ClickPipelineConte await pipeline_context.get_dagger_client(pipeline_name="Check all languages") log_options = LogOptions( list_errors=list_errors, - help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks.", + help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks. Run `airbyte-ci format fix all` for a best effort fix.", ) await run_all_subcommands(ctx, log_options) From 07b2c6dd66bd249383e9ab0b1824f78487af7294 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 14:04:48 +0100 Subject: [PATCH 124/141] do not use reserved function name for license --- .../pipelines/pipelines/airbyte_ci/format/check/commands.py | 4 ++-- .../pipelines/pipelines/airbyte_ci/format/fix/commands.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index c18a006dc130..90459e9317a1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -59,9 +59,9 @@ async def js(pipeline_context: ClickPipelineContext): await run_check(container, check_commands) -@check.command() +@check.command("license") @pass_pipeline_context -async def license(pipeline_context: ClickPipelineContext): +async def license_check(pipeline_context: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check license header") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index f101cc28c4c1..e68341d623da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# import asyncclick as click from pipelines.airbyte_ci.format.actions import run_format @@ -51,10 +53,10 @@ async def js(ctx: ClickPipelineContext): await run_format(container, format_commands) -@fix.command() +@fix.command("license") @pass_pipeline_context @click_ignore_unused_kwargs -async def license(ctx: ClickPipelineContext): +async def license_fix(ctx: ClickPipelineContext): """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" dagger_client = await ctx.get_dagger_client(pipeline_name="Add license") From e7b2b7849bf2b487ce5e8f3f67991189015cb965 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 14:09:21 +0100 Subject: [PATCH 125/141] python image as const --- .../pipelines/airbyte_ci/format/containers.py | 19 ++++--------------- .../connectors/pipelines/pipelines/consts.py | 1 + 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 71a047c47df9..85c741915bbf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -1,16 +1,13 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from typing import Any, Dict, List, Optional -import click import dagger -from pipelines.airbyte_ci.format.actions import mount_repo_for_formatting from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST -from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, NODE_IMAGE -from pipelines.dagger.actions.python.pipx import with_installed_pipx_package, with_pipx -from pipelines.dagger.containers.python import with_python_base +from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, NODE_IMAGE, PYTHON_3_10_IMAGE from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext def build_container( @@ -106,17 +103,9 @@ def format_license_container(dagger_client: dagger.Client, license_file: str) -> def format_python_container(dagger_client: dagger.Client) -> dagger.Container: """Format python code via black and isort.""" - # Here's approximately what it would look like if we built it the other way. Doesn't - # quite work. Not sure if its putting more work into. - - # base_container = with_python_base(ctx._click_context, python_version="3.10.13") - # container = with_installed_pipx_package(ctx._click_context, base_container, "poetry") - # container = mount_repo_for_formatting(container, include=["**/*.py", "pyproject.toml", "poetry.lock"]) - # return container - return build_container( dagger_client, - base_image="python:3.10.13-slim", + base_image=PYTHON_3_10_IMAGE, env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, include=["**/*.py", "pyproject.toml", "poetry.lock"], install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 196ce5b6003a..ddfbbb841298 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -32,6 +32,7 @@ AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" NODE_IMAGE = "node:18.18.0-slim" GO_IMAGE = "golang:1.17" +PYTHON_3_10_IMAGE = "python:3.10.13-slim" DOCKER_VERSION = "24.0.2" DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli" From ee74dbfb29cbd9f304b48b50d54c3da698445e4c Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 9 Nov 2023 15:06:23 +0100 Subject: [PATCH 126/141] always assign dagger_client to the same 'og' context --- .../pipelines/models/contexts/click_pipeline_context.py | 4 +++- .../connectors/pipelines/pipelines/models/singleton.py | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index d0db6bb8e2c8..619f6c6fa1e2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -26,6 +26,7 @@ class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) + _og_click_context: Callable[[], Context] = PrivateAttr(default=None) @property def params(self): @@ -67,6 +68,7 @@ def __init__(self, **data: dict[str, Any]): if not Singleton._initialized[ClickPipelineContext]: super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True + self._og_click_context = self._click_context() _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) @@ -86,7 +88,7 @@ async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> Client Avoid using this client across multiple thread pools, as it can lead to errors. Cross-thread pool calls are generally considered an anti-pattern. """ - self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + self._dagger_client = await self._og_click_context.with_async_resource(connection) # type: ignore assert self._dagger_client, "Error initializing Dagger client" return self._dagger_client.pipeline(pipeline_name) if pipeline_name else self._dagger_client diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 2d884dc136ed..349d9a46f995 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,6 +1,7 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -import threading from typing import Any, Type @@ -17,11 +18,9 @@ class Singleton: _instances: dict[Type["Singleton"], Any] = {} _initialized: dict[Type["Singleton"], bool] = {} - _lock = threading.Lock() def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: - with cls._lock: - cls._instances[cls] = super().__new__(cls) - cls._initialized[cls] = False + cls._instances[cls] = super().__new__(cls) + cls._initialized[cls] = False return cls._instances[cls] From f0b502d47bd0893a993dee40b9a86af79de2ae25 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 9 Nov 2023 22:08:40 +0100 Subject: [PATCH 127/141] format: introduce CommandResult, reuse step status, make concurrent command invokation more generic (#32296) --- .../pipelines/airbyte_ci/format/actions.py | 8 +- .../airbyte_ci/format/check/commands.py | 65 +++++++++------ .../airbyte_ci/format/fix/commands.py | 61 +++++++++++--- .../pipelines/pipelines/helpers/cli.py | 79 ++++++++----------- .../pipelines/pipelines/models/steps.py | 27 +++++-- 5 files changed, 152 insertions(+), 88 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py index 690d531b38ae..b7beab00e2e3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from typing import List @@ -6,7 +8,7 @@ from pipelines.helpers.utils import sh_dash_c -async def run_check( +def run_check( container: dagger.Container, check_commands: List[str], ) -> dagger.Container: @@ -15,7 +17,7 @@ async def run_check( container: (dagger.Container): The container to run the formatting check in check_commands (List[str]): The list of commands to run to check the formatting """ - await container.with_exec(sh_dash_c(check_commands), skip_entrypoint=True) + return container.with_exec(sh_dash_c(check_commands), skip_entrypoint=True) async def run_format( @@ -28,7 +30,7 @@ async def run_format( format_commands (List[str]): The list of commands to run to format the repository """ format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) - await format_container.directory("/src").export(".") + return await format_container.directory("/src").export(".") def mount_repo_for_formatting( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 90459e9317a1..bd4e1469b3e3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -3,7 +3,10 @@ # +import logging + import asyncclick as click +import dagger from pipelines.airbyte_ci.format.actions import run_check from pipelines.airbyte_ci.format.containers import ( format_java_container, @@ -11,8 +14,18 @@ format_license_container, format_python_container, ) -from pipelines.helpers.cli import LogOptions, run_all_subcommands +from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import CommandResult, StepStatus + + +# HELPERS +async def get_check_command_result(click_command: click.Command, checks_commands, container) -> CommandResult: + try: + stdout = await run_check(container, checks_commands).stdout() + return CommandResult(click_command, status=StepStatus.SUCCESS, stdout=stdout) + except dagger.ExecError as e: + return CommandResult(click_command, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) @click.group( @@ -26,27 +39,47 @@ async def check(): @check.command(name="all") @click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") @click.pass_context -@pass_pipeline_context -async def all_languages(ctx: click.Context, pipeline_context: ClickPipelineContext, list_errors: bool): +async def all_checks(ctx: click.Context, list_errors: bool): """ Run all format checks and fail if any checks fail. """ - await pipeline_context.get_dagger_client(pipeline_name="Check all languages") + commands_to_invoke = [command for command_name, command in check.commands.items() if command_name != ctx.command.name] + command_results = await invoke_commands_concurrently(ctx, commands_to_invoke) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + parent_command = ctx.parent.command + logger = logging.getLogger(parent_command.name) log_options = LogOptions( list_errors=list_errors, help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks. Run `airbyte-ci format fix all` for a best effort fix.", ) - await run_all_subcommands(ctx, log_options) + log_command_results(ctx, logger, log_options, command_results) + if failure: + raise click.Abort() @check.command() @pass_pipeline_context -async def java(pipeline_context: ClickPipelineContext): +async def python(pipeline_context: ClickPipelineContext) -> CommandResult: + """Format python code via black and isort.""" + + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check python formatting") + container = format_python_container(dagger_client) + check_commands = [ + "poetry install --no-root", + "poetry run isort --settings-file pyproject.toml --check-only .", + "poetry run black --config pyproject.toml --check .", + ] + return await get_check_command_result(check.commands["python"], check_commands, container) + + +@check.command() +@pass_pipeline_context +async def java(pipeline_context: ClickPipelineContext) -> CommandResult: """Format java, groovy, and sql code via spotless.""" dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check java formatting") container = format_java_container(dagger_client) check_commands = ["./gradlew spotlessCheck --scan"] - await run_check(container, check_commands) + return await get_check_command_result(check.commands["java"], check_commands, container) @check.command() @@ -56,7 +89,7 @@ async def js(pipeline_context: ClickPipelineContext): dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check js formatting") container = format_js_container(dagger_client) check_commands = ["prettier --check ."] - await run_check(container, check_commands) + return await get_check_command_result(check.commands["js"], check_commands, container) @check.command("license") @@ -67,18 +100,4 @@ async def license_check(pipeline_context: ClickPipelineContext): dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check license header") container = format_license_container(dagger_client, license_file) check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] - await run_check(container, check_commands) - - -@check.command() -@pass_pipeline_context -async def python(pipeline_context: ClickPipelineContext): - """Format python code via black and isort.""" - dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check python formatting") - container = format_python_container(dagger_client) - check_commands = [ - "poetry install --no-root", - "poetry run isort --settings-file pyproject.toml --check-only .", - "poetry run black --config pyproject.toml --check .", - ] - await run_check(container, check_commands) + return await get_check_command_result(check.commands["license"], check_commands, container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index e68341d623da..c9210626ddaa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -2,7 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import List + import asyncclick as click +import dagger from pipelines.airbyte_ci.format.actions import run_format from pipelines.airbyte_ci.format.containers import ( format_java_container, @@ -11,8 +14,32 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.cli import run_all_subcommands +from pipelines.helpers.cli import invoke_commands_concurrently, invoke_commands_sequentially from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import CommandResult, StepStatus + +# HELPERS +LANGUAGE_FIX_COMMAND_NAMES = ["python", "java", "js"] + + +async def get_format_command_result(click_command: click.Command, container: dagger.Container, format_commands: List[str]) -> CommandResult: + """Run a format command and return the CommandResult. + A command is considered successful if the export operation of run_format is successful. + + Args: + click_command (click.Command): The click command to run + container (dagger.Container): The container to run the format_commands in + format_commands (List[str]): The list of commands to run to format the repository + + Returns: + CommandResult: The result of running the command + """ + try: + successful_export = await run_format(container, format_commands) + status = StepStatus.SUCCESS if successful_export else StepStatus.FAILURE + return CommandResult(click_command, status=status) + except dagger.ExecError as e: + return CommandResult(click_command, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) @click.group( @@ -25,50 +52,58 @@ async def fix(): @fix.command(name="all") @click.pass_context -@pass_pipeline_context -async def all_languages(ctx: click.Context, pipeline_context: ClickPipelineContext): +async def all_fix(ctx: click.Context): """Run code format checks and fix any failures.""" - await pipeline_context.get_dagger_client(pipeline_name="Fix all languages") - await run_all_subcommands(ctx) + # We can run language commands concurrently because they modify different set of files. + commands_to_invoke_concurrently = [fix.commands[command_name] for command_name in LANGUAGE_FIX_COMMAND_NAMES] + command_results = await invoke_commands_concurrently(ctx, commands_to_invoke_concurrently) + + # We have to run license command sequentially because it modifies the same set of files as other commands. + # If we ran it concurrently with language commands, we face race condition issues. + command_results += await invoke_commands_sequentially(ctx, [fix.commands["license"]]) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + + if failure: + raise click.Abort() @fix.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def java(ctx: ClickPipelineContext): +async def java(ctx: ClickPipelineContext) -> CommandResult: """Format java, groovy, and sql code via spotless.""" dagger_client = await ctx.get_dagger_client(pipeline_name="Format java") container = format_java_container(dagger_client) format_commands = ["./gradlew spotlessApply --scan"] - await run_format(container, format_commands) + return await get_format_command_result(fix.commands["java"], container, format_commands) @fix.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def js(ctx: ClickPipelineContext): +async def js(ctx: ClickPipelineContext) -> CommandResult: dagger_client = await ctx.get_dagger_client(pipeline_name="Format js") container = format_js_container(dagger_client) format_commands = ["prettier --write ."] - await run_format(container, format_commands) + return await get_format_command_result(fix.commands["js"], container, format_commands) @fix.command("license") @pass_pipeline_context @click_ignore_unused_kwargs -async def license_fix(ctx: ClickPipelineContext): +async def license_fix(ctx: ClickPipelineContext) -> CommandResult: """Add license to python and java code via addlicense.""" license_file = "LICENSE_SHORT" dagger_client = await ctx.get_dagger_client(pipeline_name="Add license") container = format_license_container(dagger_client, license_file) format_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."] - await run_format(container, format_commands) + return await get_format_command_result(fix.commands["license"], container, format_commands) @fix.command() @pass_pipeline_context @click_ignore_unused_kwargs -async def python(ctx: ClickPipelineContext): +async def python(ctx: ClickPipelineContext) -> CommandResult: """Format python code via black and isort.""" dagger_client = await ctx.get_dagger_client(pipeline_name="Format python") container = format_python_container(dagger_client) @@ -77,4 +112,4 @@ async def python(ctx: ClickPipelineContext): "poetry run isort --settings-file pyproject.toml .", "poetry run black --config pyproject.toml .", ] - await run_format(container, format_commands) + return await get_format_command_result(fix.commands["python"], container, format_commands) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index c78a243a39c0..9a02a2b8088f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -1,29 +1,30 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + -import logging -import sys from dataclasses import dataclass -from typing import Dict, Tuple +from typing import Any, List -import anyio import asyncclick as click -import dagger +import asyncer from jinja2 import Template +from pipelines.models.steps import CommandResult, StepStatus ALL_RESULTS_KEY = "_run_all_results" SUMMARY_TEMPLATE_STR = """ -Summary of Sub Command Results +Summary of commands results ======================== -{% for command_name, result in results.items() %} -{{ '✅' if result[0] else '❌' }} {{ command_prefix }} {{ command_name }} +{% for command_name, success in results %} +{{ '✅' if success else '❌' }} {{ command_prefix }} {{ command_name }} {% endfor %} """ DETAILS_TEMPLATE_STR = """ -Detailed Errors for Failed Sub Commands +Detailed Errors for failed commands ================================= {% for command_name, error in failed_commands_details %} ❌ {{ command_prefix }} {{ command_name }} @@ -41,22 +42,27 @@ class LogOptions: help_message: str = None -def _log_output(ctx: click.Context, logger, options: LogOptions = LogOptions()): +def log_command_results(ctx: click.Context, logger, options: LogOptions = LogOptions(), command_results=List[CommandResult]): """ Log the output of the subcommands run by `run_all_subcommands`. """ - subcommand_results = ctx.obj[ALL_RESULTS_KEY] command_path = ctx.command_path summary_template = Template(SUMMARY_TEMPLATE_STR) - summary_message = summary_template.render(results=subcommand_results, command_prefix=command_path) + summary_message = summary_template.render( + results=[(r.command.name, r.status is StepStatus.SUCCESS) for r in command_results], command_prefix=command_path + ) logger.info(summary_message) - result_contains_failures = any(not succeeded for (succeeded, _) in subcommand_results.values()) + result_contains_failures = any([r.status is StepStatus.FAILURE for r in command_results]) if result_contains_failures: if options.list_errors: - failed_commands_details = [(name, error) for name, (success, error) in subcommand_results.items() if not success] + failed_commands_details = [ + (command_result.command.name, command_result.stderr) + for command_result in command_results + if command_result.status is StepStatus.FAILURE + ] if failed_commands_details: details_template = Template(DETAILS_TEMPLATE_STR) details_message = details_template.render(failed_commands_details=failed_commands_details, command_prefix=command_path) @@ -68,39 +74,24 @@ def _log_output(ctx: click.Context, logger, options: LogOptions = LogOptions()): logger.info(options.help_message) -async def _run_sub_command(ctx: click.Context, command: click.Command): +async def invoke_commands_concurrently(ctx: click.Context, commands: List[click.Command]) -> List[Any]: """ - Run a subcommand and store the result in the context object. + Run click commands concurrently and return a list of their return values. """ - try: - await ctx.invoke(command) - ctx.obj[ALL_RESULTS_KEY][command.name] = (True, None) - except dagger.ExecError as e: - ctx.obj[ALL_RESULTS_KEY][command.name] = (False, str(e)) + + soon_command_executions_results = [] + async with asyncer.create_task_group() as command_group: + for command in commands: + soon_command_execution_result = command_group.soonify(ctx.invoke)(command) + soon_command_executions_results.append(soon_command_execution_result) + return [r.value for r in soon_command_executions_results] -async def run_all_subcommands(ctx: click.Context, log_options: LogOptions = LogOptions()): +async def invoke_commands_sequentially(ctx: click.Context, commands: List[click.Command]) -> List[Any]: """ - Run all subcommands of a given command and log the results. + Run click commands sequentially and return a list of their return values. """ - ctx.obj[ALL_RESULTS_KEY] = {} - - parent_command_path = ctx.parent.command_path - parent_command_name = ctx.parent.command.name - current_command_name = ctx.command.name - - logger = logging.getLogger(parent_command_name) - - # omit current command from list of subcommands - all_subcommands_dict = ctx.parent.command.commands - filtered_subcommands_dict = {name: command for name, command in all_subcommands_dict.items() if name != current_command_name} - - logger.info(f"Running all sub commands of {parent_command_path}...") - async with anyio.create_task_group() as check_group: - for command in filtered_subcommands_dict.values(): - check_group.start_soon(_run_sub_command, ctx, command) - - _log_output(ctx, logger, log_options) - - if any(not succeeded for (succeeded, _) in ctx.obj[ALL_RESULTS_KEY].values()): - sys.exit(1) + command_executions_results = [] + for command in commands: + command_executions_results.append(await ctx.invoke(command)) + return command_executions_results diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index 5cc119ed6440..9959653cdf4f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -5,26 +5,25 @@ from __future__ import annotations import logging -import typing from abc import abstractmethod from dataclasses import dataclass, field from datetime import datetime, timedelta from enum import Enum from pathlib import Path -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import anyio import asyncer +import click from dagger import Container, DaggerError from pipelines import main_logger from pipelines.helpers import sentry_utils from pipelines.helpers.utils import format_duration, get_exec_result -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from pipelines.models.contexts.pipeline_context import PipelineContext from abc import ABC -from typing import ClassVar from rich.style import Style @@ -62,7 +61,7 @@ def is_file(self) -> bool: class StepResult: """A dataclass to capture the result of a step.""" - step: Step + step: Union[Step, click.command] status: StepStatus created_at: datetime = field(default_factory=datetime.utcnow) stderr: Optional[str] = None @@ -88,6 +87,24 @@ def redact_secrets_from_string(self, value: str) -> str: return value +@dataclass(frozen=True) +class CommandResult: + """A dataclass to capture the result of a command.""" + + command: click.command + status: StepStatus + created_at: datetime = field(default_factory=datetime.utcnow) + stderr: Optional[str] = None + stdout: Optional[str] = None + exc_info: Optional[Exception] = None + + def __repr__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + class StepStatus(Enum): """An Enum to characterize the success, failure or skipping of a Step.""" From 49b44a1a5d4a59abae69e4dbbc656e6bd372101a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 9 Nov 2023 17:45:50 -0700 Subject: [PATCH 128/141] Add sibling command helper --- .../airbyte_ci/format/check/commands.py | 14 ++++++++------ .../airbyte_ci/format/fix/commands.py | 18 ++++++++++-------- .../pipelines/pipelines/helpers/cli.py | 6 ++++++ .../models/contexts/click_pipeline_context.py | 4 ++++ .../source_klaviyo/exceptions.py | 2 ++ 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index bd4e1469b3e3..1cac37c2b799 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -14,7 +14,8 @@ format_license_container, format_python_container, ) -from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, log_command_results +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.cli import LogOptions, get_all_sibling_commands, invoke_commands_concurrently, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import CommandResult, StepStatus @@ -43,8 +44,9 @@ async def all_checks(ctx: click.Context, list_errors: bool): """ Run all format checks and fail if any checks fail. """ - commands_to_invoke = [command for command_name, command in check.commands.items() if command_name != ctx.command.name] - command_results = await invoke_commands_concurrently(ctx, commands_to_invoke) + all_commands = get_all_sibling_commands(ctx) + + command_results = await invoke_commands_concurrently(ctx, all_commands) failure = any([r.status is StepStatus.FAILURE for r in command_results]) parent_command = ctx.parent.command logger = logging.getLogger(parent_command.name) @@ -57,7 +59,7 @@ async def all_checks(ctx: click.Context, list_errors: bool): raise click.Abort() -@check.command() +@check.command(cls=DaggerPipelineCommand) @pass_pipeline_context async def python(pipeline_context: ClickPipelineContext) -> CommandResult: """Format python code via black and isort.""" @@ -72,7 +74,7 @@ async def python(pipeline_context: ClickPipelineContext) -> CommandResult: return await get_check_command_result(check.commands["python"], check_commands, container) -@check.command() +@check.command(cls=DaggerPipelineCommand) @pass_pipeline_context async def java(pipeline_context: ClickPipelineContext) -> CommandResult: """Format java, groovy, and sql code via spotless.""" @@ -82,7 +84,7 @@ async def java(pipeline_context: ClickPipelineContext) -> CommandResult: return await get_check_command_result(check.commands["java"], check_commands, container) -@check.command() +@check.command(cls=DaggerPipelineCommand) @pass_pipeline_context async def js(pipeline_context: ClickPipelineContext): """Format yaml and json code via prettier.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index c9210626ddaa..f242bcc7d5a7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -14,12 +14,13 @@ format_python_container, ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs -from pipelines.helpers.cli import invoke_commands_concurrently, invoke_commands_sequentially +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.cli import invoke_commands_concurrently, invoke_commands_sequentially, get_all_sibling_commands from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import CommandResult, StepStatus # HELPERS -LANGUAGE_FIX_COMMAND_NAMES = ["python", "java", "js"] + async def get_format_command_result(click_command: click.Command, container: dagger.Container, format_commands: List[str]) -> CommandResult: @@ -50,13 +51,14 @@ async def fix(): pass -@fix.command(name="all") +@fix.command(cls=DaggerPipelineCommand, name="all") @click.pass_context async def all_fix(ctx: click.Context): """Run code format checks and fix any failures.""" + sibling_commands = get_all_sibling_commands(ctx) + # We can run language commands concurrently because they modify different set of files. - commands_to_invoke_concurrently = [fix.commands[command_name] for command_name in LANGUAGE_FIX_COMMAND_NAMES] - command_results = await invoke_commands_concurrently(ctx, commands_to_invoke_concurrently) + command_results = await invoke_commands_concurrently(ctx, sibling_commands) # We have to run license command sequentially because it modifies the same set of files as other commands. # If we ran it concurrently with language commands, we face race condition issues. @@ -67,7 +69,7 @@ async def all_fix(ctx: click.Context): raise click.Abort() -@fix.command() +@fix.command(cls=DaggerPipelineCommand) @pass_pipeline_context @click_ignore_unused_kwargs async def java(ctx: ClickPipelineContext) -> CommandResult: @@ -78,7 +80,7 @@ async def java(ctx: ClickPipelineContext) -> CommandResult: return await get_format_command_result(fix.commands["java"], container, format_commands) -@fix.command() +@fix.command(cls=DaggerPipelineCommand) @pass_pipeline_context @click_ignore_unused_kwargs async def js(ctx: ClickPipelineContext) -> CommandResult: @@ -100,7 +102,7 @@ async def license_fix(ctx: ClickPipelineContext) -> CommandResult: return await get_format_command_result(fix.commands["license"], container, format_commands) -@fix.command() +@fix.command(cls=DaggerPipelineCommand) @pass_pipeline_context @click_ignore_unused_kwargs async def python(ctx: ClickPipelineContext) -> CommandResult: diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index 9a02a2b8088f..c56e35ce5c88 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -95,3 +95,9 @@ async def invoke_commands_sequentially(ctx: click.Context, commands: List[click. for command in commands: command_executions_results.append(await ctx.invoke(command)) return command_executions_results + +def get_all_sibling_commands(ctx: click.Context) -> List[click.Command]: + """ + Get all sibling commands of the current command. + """ + return [c for c in ctx.parent.command.commands.values() if c.name != ctx.command.name] diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 619f6c6fa1e2..6f9878f38f5c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -68,6 +68,10 @@ def __init__(self, **data: dict[str, Any]): if not Singleton._initialized[ClickPipelineContext]: super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True + + """ + Note: Its important to hold onto the original click context object, as it is used to hold onto the Dagger client. + """ self._og_click_context = self._click_context() _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py index f68eecb35695..d0321754618d 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py @@ -1,2 +1,4 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + class KlaviyoBackoffError(Exception): """An exception which is raised when 'retry-after' time is longer than 'max_time' specified""" From 720ec470f3eecb3e64233eeb723e206775f63036 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 9 Nov 2023 18:11:41 -0700 Subject: [PATCH 129/141] Fix fix command --- .../airbyte_ci/format/check/commands.py | 2 +- .../airbyte_ci/format/fix/commands.py | 31 ++++++++++++++----- .../pipelines/pipelines/helpers/cli.py | 9 +++--- .../source_klaviyo/exceptions.py | 1 + 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py index 1cac37c2b799..4b97275cfe57 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -54,7 +54,7 @@ async def all_checks(ctx: click.Context, list_errors: bool): list_errors=list_errors, help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks. Run `airbyte-ci format fix all` for a best effort fix.", ) - log_command_results(ctx, logger, log_options, command_results) + log_command_results(ctx, command_results, logger, log_options) if failure: raise click.Abort() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py index f242bcc7d5a7..3667ab8882d1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from typing import List import asyncclick as click @@ -15,14 +16,19 @@ ) from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from pipelines.helpers.cli import invoke_commands_concurrently, invoke_commands_sequentially, get_all_sibling_commands +from pipelines.helpers.cli import ( + LogOptions, + get_all_sibling_commands, + invoke_commands_concurrently, + invoke_commands_sequentially, + log_command_results, +) from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import CommandResult, StepStatus # HELPERS - async def get_format_command_result(click_command: click.Command, container: dagger.Container, format_commands: List[str]) -> CommandResult: """Run a format command and return the CommandResult. A command is considered successful if the export operation of run_format is successful. @@ -55,18 +61,29 @@ async def fix(): @click.pass_context async def all_fix(ctx: click.Context): """Run code format checks and fix any failures.""" - sibling_commands = get_all_sibling_commands(ctx) + parent_command = ctx.parent.command + logger = logging.getLogger(parent_command.name) + + concurrent_commands = [ + fix.commands["python"], + fix.commands["java"], + fix.commands["js"], + ] + sequential_commands = [fix.commands["license"]] # We can run language commands concurrently because they modify different set of files. - command_results = await invoke_commands_concurrently(ctx, sibling_commands) + command_results = await invoke_commands_concurrently(ctx, concurrent_commands) # We have to run license command sequentially because it modifies the same set of files as other commands. # If we ran it concurrently with language commands, we face race condition issues. - command_results += await invoke_commands_sequentially(ctx, [fix.commands["license"]]) + command_results += await invoke_commands_sequentially(ctx, sequential_commands) failure = any([r.status is StepStatus.FAILURE for r in command_results]) - if failure: - raise click.Abort() + log_options = LogOptions(list_errors=True) + + log_command_results(ctx, command_results, logger, log_options) + + return not failure @fix.command(cls=DaggerPipelineCommand) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py index c56e35ce5c88..2aaa77258a7f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -4,6 +4,7 @@ from dataclasses import dataclass +from logging import Logger from typing import Any, List import asyncclick as click @@ -42,16 +43,15 @@ class LogOptions: help_message: str = None -def log_command_results(ctx: click.Context, logger, options: LogOptions = LogOptions(), command_results=List[CommandResult]): +def log_command_results(ctx: click.Context, command_results: List[CommandResult], logger: Logger, options: LogOptions = LogOptions()): """ Log the output of the subcommands run by `run_all_subcommands`. """ command_path = ctx.command_path summary_template = Template(SUMMARY_TEMPLATE_STR) - summary_message = summary_template.render( - results=[(r.command.name, r.status is StepStatus.SUCCESS) for r in command_results], command_prefix=command_path - ) + results = [(r.command.name, r.status is StepStatus.SUCCESS) for r in command_results] + summary_message = summary_template.render(results=results, command_prefix=command_path) logger.info(summary_message) result_contains_failures = any([r.status is StepStatus.FAILURE for r in command_results]) @@ -96,6 +96,7 @@ async def invoke_commands_sequentially(ctx: click.Context, commands: List[click. command_executions_results.append(await ctx.invoke(command)) return command_executions_results + def get_all_sibling_commands(ctx: click.Context) -> List[click.Command]: """ Get all sibling commands of the current command. diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py index d0321754618d..63df81365345 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py @@ -1,4 +1,5 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. + class KlaviyoBackoffError(Exception): """An exception which is raised when 'retry-after' time is longer than 'max_time' specified""" From a6a1d3779e4d3f9c7008f25b198fd1784c68e925 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Nov 2023 11:24:22 +0100 Subject: [PATCH 130/141] format additional files --- .../unit_tests/destinations/vector_db_based/config_test.py | 4 ++++ .../connectors/source-intercom/unit_tests/test_source.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py index 468d7cc61d51..c6ccf6da1985 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from typing import Union import dpath.util diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py index 0cf50ccbc171..fe7a765a2ee0 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from source_intercom import SourceIntercom From 76f83ea8ca16afcbc5ad8cf86618d2e47308f498 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Fri, 10 Nov 2023 12:14:06 +0100 Subject: [PATCH 131/141] remove format from gradle.yml --- .github/workflows/gradle.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8ada6b1552d2..13fa98a6acaa 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -103,8 +103,7 @@ jobs: with: read-only: ${{ github.ref != 'refs/heads/master' }} # TODO: be able to remove the skipSlowTests property - # TODO: remove the format task ASAP - arguments: --scan --no-daemon --no-watch-fs format check -DskipSlowTests=true + arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true # In case of self-hosted EC2 errors, remove this block. stop-check-runner: From 0bcf0fcfa735694134af777eec40ad7a9ef747fb Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 10 Nov 2023 09:51:42 -0500 Subject: [PATCH 132/141] gradle: tweak spotless config --- .../pipelines/airbyte_ci/format/containers.py | 4 +--- build.gradle | 17 +++++++---------- tools/gradle/codestyle/sql-dbeaver.properties | 8 -------- 3 files changed, 8 insertions(+), 21 deletions(-) delete mode 100644 tools/gradle/codestyle/sql-dbeaver.properties diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py index 85c741915bbf..f9414b22edfd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -62,7 +62,6 @@ def format_java_container(dagger_client: dagger.Client) -> dagger.Container: base_image=AMAZONCORRETTO_IMAGE, include=[ "**/*.java", - "**/*.sql", "**/*.gradle", "gradlew", "gradlew.bat", @@ -71,7 +70,6 @@ def format_java_container(dagger_client: dagger.Client) -> dagger.Container: "**/gradle.properties", "**/version.properties", "tools/gradle/codestyle/java-google-style.xml", - "tools/gradle/codestyle/sql-dbeaver.properties", ], install_commands=[ "yum update -y", @@ -87,7 +85,7 @@ def format_js_container(dagger_client: dagger.Client) -> dagger.Container: dagger_client, base_image=NODE_IMAGE, include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], - install_commands=["npm install -g npm@10.1.0", "npm install -g prettier@2.8.1"], + install_commands=["npm install -g npm@10.1.0 prettier@3.0.3"], ) diff --git a/build.gradle b/build.gradle index 8fd46fd9c1b3..e6543d9ded8e 100644 --- a/build.gradle +++ b/build.gradle @@ -138,19 +138,16 @@ allprojects { target groovyGradleTarget } } - def sqlTarget = createFormatTarget('**/*.sql') - if (!sqlTarget.isEmpty()) { - sql { - target sqlTarget - dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) - } + } + + if (rootProject.ext.skipSlowTests) { + // Format checks should be run via airbyte-ci. + tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { + enabled = false } } } -// TODO: remove this once the CI routinely enforces formatting checks. -tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { - enabled = false -} + def getCDKTargetVersion() { def props = new Properties() diff --git a/tools/gradle/codestyle/sql-dbeaver.properties b/tools/gradle/codestyle/sql-dbeaver.properties deleted file mode 100644 index cecd2e3813c6..000000000000 --- a/tools/gradle/codestyle/sql-dbeaver.properties +++ /dev/null @@ -1,8 +0,0 @@ -# case of the keywords (UPPER, LOWER or ORIGINAL) -sql.formatter.keyword.case=UPPER -# Statement delimiter -sql.formatter.statement.delimiter=; -# Indentation style (space or tab) -sql.formatter.indent.type=space -# Number of identation characters -sql.formatter.indent.size=4 From c48e9c9162d7b0f0a766dbfd751d8c82b27fdd0d Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 10 Nov 2023 10:41:38 -0500 Subject: [PATCH 133/141] remove octavia-cli from gradle build --- octavia-cli/build.gradle | 22 ---------------------- settings.gradle | 1 - 2 files changed, 23 deletions(-) delete mode 100644 octavia-cli/build.gradle diff --git a/octavia-cli/build.gradle b/octavia-cli/build.gradle deleted file mode 100644 index 4fb075a628b1..000000000000 --- a/octavia-cli/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - id "org.openapi.generator" version "5.3.1" - id 'airbyte-python' - id 'airbyte-docker-legacy' -} - -def generateApiClient = tasks.register('generateApiClient', GenerateTask) { - inputSpec = "$rootDir.absolutePath/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml" - outputDir = "$buildDir/airbyte_api_client" - - generatorName = "python" - packageName = "airbyte_api_client" -} -tasks.register('generate').configure { - dependsOn generateApiClient -} - -//tasks.named('installReqs').configure { -// dependsOn generateApiClient -//} diff --git a/settings.gradle b/settings.gradle index c7e237404a43..91cb8e6871f1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -160,7 +160,6 @@ if (isCiServer || isAirbyteCI) { rootProject.name = 'airbyte' include ':tools:code-generator' -include ':octavia-cli' include ':airbyte-cdk:python' include ':airbyte-cdk:java:airbyte-cdk' From 749ffabc478db548d1f3e912490e44de1c09279a Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 10 Nov 2023 11:06:20 -0500 Subject: [PATCH 134/141] gradle: restore commented block in base-normalization --- .../bases/base-normalization/build.gradle | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 0c733d7cda16..5b4d5de4355b 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -38,26 +38,26 @@ generate.configure { tasks.named('check').configure { dependsOn generate } -// -//[ -// 'bigquery', -// 'mysql', -// 'postgres', -// 'redshift', -// 'snowflake', -// 'oracle', -// 'mssql', -// 'clickhouse', -// 'tidb', -// 'duckdb', -//].each {destinationName -> -// def integrationTestPython = tasks.named('integrationTestPython') -// integrationTestPython.configure { -// dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') -// } -// // Not really sure what this task does differently from customIntegrationTestPython, -// // but it seems to also run integration tests and as such it depends on the docker images. -// integrationTestPython.configure { -// dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') -// } -//} + +[ + 'bigquery', + 'mysql', + 'postgres', + 'redshift', + 'snowflake', + 'oracle', + 'mssql', + 'clickhouse', + 'tidb', + 'duckdb', +].each {destinationName -> + def integrationTestPython = tasks.named('integrationTestPython') + integrationTestPython.configure { + dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') + } + // Not really sure what this task does differently from customIntegrationTestPython, + // but it seems to also run integration tests and as such it depends on the docker images. + integrationTestPython.configure { + dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') + } +} From 139f8094aabf5313e9a44a599923956a21f61e11 Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Fri, 10 Nov 2023 11:26:46 -0500 Subject: [PATCH 135/141] fix base-normalization issue --- .../bases/base-normalization/build.gradle | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 5b4d5de4355b..961ff66d0097 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -51,13 +51,7 @@ tasks.named('check').configure { 'tidb', 'duckdb', ].each {destinationName -> - def integrationTestPython = tasks.named('integrationTestPython') - integrationTestPython.configure { - dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') - } - // Not really sure what this task does differently from customIntegrationTestPython, - // but it seems to also run integration tests and as such it depends on the docker images. - integrationTestPython.configure { + tasks.matching { it.name == 'integrationTestPython' }.configureEach { dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') } } From 84ff3b5df514bd5338e11d7614321229f94218ba Mon Sep 17 00:00:00 2001 From: Ben Church Date: Fri, 10 Nov 2023 13:01:32 -0700 Subject: [PATCH 136/141] Add auto format --- .github/workflows/format.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index fa90a300b2a2..54314e1a73df 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,9 +12,9 @@ on: - master pull_request: jobs: - format-check: + format-fix: runs-on: "conn-prod-xlarge-runner" - name: "Check All Formatting Rules" + name: "Apply All Formatting Rules" timeout-minutes: 40 steps: - name: Checkout Airbyte @@ -44,7 +44,7 @@ jobs: - '**/*' - '!**/*.md' - - name: Run airbyte-ci formatting checks + - name: Run airbyte-ci format fix uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" @@ -53,13 +53,27 @@ jobs: gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "format check all --list-errors" + subcommand: "format fix all" + + + # This is helpful in the case that we change a previously committed generated file to be ignored by git. + - name: Remove any files that have been gitignored + run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached + + - name: Commit Formatting Changes (PR) + uses: stefanzweifel/git-auto-commit-action@v4 + # do not commit if master branch + if: github.ref != 'refs/heads/master' + with: + commit_message: Automated Commit - Formatting Changes + commit_user_name: Octavia Squidington III + commit_user_email: octavia-squidington-iii@users.noreply.github.com notify-failure-slack-channel: name: "Notify Slack Channel on Build Failures" runs-on: ubuntu-latest needs: - - format-check + - format-fix if: ${{ failure() && github.ref == 'refs/heads/master' }} steps: - name: Checkout Airbyte From 89f1b2c5b9e084cb62421e457b1a0789db470cc4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 13 Nov 2023 09:59:05 +0100 Subject: [PATCH 137/141] bump git-auto-commit-action to v5 --- .github/workflows/format.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 54314e1a73df..e826618861d5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -55,13 +55,12 @@ jobs: github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} subcommand: "format fix all" - # This is helpful in the case that we change a previously committed generated file to be ignored by git. - name: Remove any files that have been gitignored run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached - name: Commit Formatting Changes (PR) - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 # do not commit if master branch if: github.ref != 'refs/heads/master' with: From 8385bcf06b482602b1eae9cbcbe86782668ad81f Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 13 Nov 2023 10:03:11 +0100 Subject: [PATCH 138/141] to revert: test format --- airbyte-integrations/connectors/source-pokeapi/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 9ec4c483ecf9..20943cae7603 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,12 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.entrypoint import launch import sys -from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi -if __name__ == "__main__": +if __name__ == '__main__': source = SourcePokeapi() launch(source, sys.argv[1:]) From 969d23f3ec5c9b45d6c66ed86a1e7c14bf050ce7 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 13 Nov 2023 09:08:46 +0000 Subject: [PATCH 139/141] Automated Commit - Formatting Changes --- airbyte-integrations/connectors/source-pokeapi/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 20943cae7603..38a510a3f2d7 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,12 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.entrypoint import launch - import sys +from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi -if __name__ == '__main__': +if __name__ == "__main__": source = SourcePokeapi() launch(source, sys.argv[1:]) From 84fd85ac7167cff3cd0514ae3da97591d427168b Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 13 Nov 2023 10:11:08 +0100 Subject: [PATCH 140/141] Revert "Automated Commit - Formatting Changes" This reverts commit 969d23f3ec5c9b45d6c66ed86a1e7c14bf050ce7. --- airbyte-integrations/connectors/source-pokeapi/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 38a510a3f2d7..20943cae7603 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,11 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.entrypoint import launch + import sys -from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi -if __name__ == "__main__": +if __name__ == '__main__': source = SourcePokeapi() launch(source, sys.argv[1:]) From 9c34b9d3675386ec7aa2f66958bee230d440cd09 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 13 Nov 2023 09:14:14 +0000 Subject: [PATCH 141/141] Automated Commit - Formatting Changes --- airbyte-integrations/connectors/source-pokeapi/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 20943cae7603..38a510a3f2d7 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,12 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.entrypoint import launch - import sys +from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi -if __name__ == '__main__': +if __name__ == "__main__": source = SourcePokeapi() launch(source, sys.argv[1:])