From 9656c4e3de83a3a62366279c6d440733981a4e2c Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 11 Jan 2024 13:08:04 -0800 Subject: [PATCH] Airbyte-ci: Ensure we set the working directory earlier (#34136) --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/pipelines/cli/airbyte_ci.py | 62 ++----------------- .../pipelines/cli/ensure_repo_root.py | 57 +++++++++++++++++ .../connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 65 insertions(+), 57 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index a0781b26c37d..4f26d9ad07fe 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -521,6 +521,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 3.1.3 | [#34136](https://github.com/airbytehq/airbyte/pull/34136) | Fix issue where dagger excludes were not being properly applied | | 3.1.2 | [#33972](https://github.com/airbytehq/airbyte/pull/33972) | Remove secrets scrubbing hack for --is-local and other small tweaks. | | 3.1.1 | [#33979](https://github.com/airbytehq/airbyte/pull/33979) | Fix AssertionError on report existence again | | 3.1.0 | [#33994](https://github.com/airbytehq/airbyte/pull/33994) | Log more context information in CI. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 028bb54df7c7..cb5226728929 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -6,16 +6,20 @@ from __future__ import annotations +# Important: This import and function call must be the first import in this file +# This is needed to ensure that the working directory is the root of the airbyte repo +from pipelines.cli.ensure_repo_root import set_working_directory_to_root + +set_working_directory_to_root() + import logging import multiprocessing import os import sys -from pathlib import Path from typing import Optional import asyncclick as click import docker # type: ignore -import git from github import PullRequest from pipelines import main_logger from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag @@ -30,58 +34,6 @@ from pipelines.helpers.utils import get_current_epoch_time -def _validate_airbyte_repo(repo: git.Repo) -> bool: - """Check if any of the remotes are the airbyte repo.""" - expected_repo_name = "airbytehq/airbyte" - for remote in repo.remotes: - if expected_repo_name in remote.url: - return True - - warning_message = f""" - ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ - - It looks like you are not running this command from the airbyte repo ({expected_repo_name}). - - If this command is run from outside the airbyte repo, it will not work properly. - - Please run this command your local airbyte project. - - ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ - """ - - logging.warning(warning_message) - - return False - - -def get_airbyte_repo() -> git.Repo: - """Get the airbyte repo.""" - repo = git.Repo(search_parent_directories=True) - _validate_airbyte_repo(repo) - return repo - - -def get_airbyte_repo_path_with_fallback() -> Path: - """Get the path to the airbyte repo.""" - try: - repo_path = get_airbyte_repo().working_tree_dir - if repo_path is not None: - return Path(str(get_airbyte_repo().working_tree_dir)) - except git.exc.InvalidGitRepositoryError: - pass - logging.warning("Could not find the airbyte repo, falling back to the current working directory.") - path = Path.cwd() - logging.warning(f"Using {path} as the airbyte repo path.") - return path - - -def set_working_directory_to_root() -> None: - """Set the working directory to the root of the airbyte repo.""" - working_dir = get_airbyte_repo_path_with_fallback() - logging.info(f"Setting working directory to {working_dir}") - os.chdir(working_dir) - - def log_context_info(ctx: click.Context) -> None: main_logger.info(f"Running airbyte-ci version {__installed_version__}") main_logger.info(f"Running dagger version {get_dagger_sdk_version()}") @@ -241,7 +193,5 @@ async def airbyte_ci(ctx: click.Context) -> None: # noqa D103 log_context_info(ctx) -set_working_directory_to_root() - if __name__ == "__main__": airbyte_ci() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py new file mode 100644 index 000000000000..7b0c4b39576a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py @@ -0,0 +1,57 @@ +import logging +import os +from pathlib import Path + +import git + + +def _validate_airbyte_repo(repo: git.Repo) -> bool: + """Check if any of the remotes are the airbyte repo.""" + expected_repo_name = "airbytehq/airbyte" + for remote in repo.remotes: + if expected_repo_name in remote.url: + return True + + warning_message = f""" + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + + It looks like you are not running this command from the airbyte repo ({expected_repo_name}). + + If this command is run from outside the airbyte repo, it will not work properly. + + Please run this command your local airbyte project. + + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + """ + + logging.warning(warning_message) + + return False + + +def get_airbyte_repo() -> git.Repo: + """Get the airbyte repo.""" + repo = git.Repo(search_parent_directories=True) + _validate_airbyte_repo(repo) + return repo + + +def get_airbyte_repo_path_with_fallback() -> Path: + """Get the path to the airbyte repo.""" + try: + repo_path = get_airbyte_repo().working_tree_dir + if repo_path is not None: + return Path(str(get_airbyte_repo().working_tree_dir)) + except git.exc.InvalidGitRepositoryError: + pass + logging.warning("Could not find the airbyte repo, falling back to the current working directory.") + path = Path.cwd() + logging.warning(f"Using {path} as the airbyte repo path.") + return path + + +def set_working_directory_to_root() -> None: + """Set the working directory to the root of the airbyte repo.""" + working_dir = get_airbyte_repo_path_with_fallback() + logging.info(f"Setting working directory to {working_dir}") + os.chdir(working_dir) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 2411a03fe1db..256d1075bad8 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.1.2" +version = "3.1.3" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]