Airbyte-ci: Generate binary files for airbyte-ci

.github/actions/run-dagger-pipeline/action.yml

@@ -104,7 +104,7 @@ runs:
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"

.github/workflows/airbyte-ci-release-experiment.yml

@@ -7,6 +7,11 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  push:
+    branches:
+      - master
+    paths:
+      - "airbyte-ci/connectors/pipelines/**"
   workflow_dispatch:
 jobs:
   build:
@@ -17,16 +22,70 @@ jobs:
         os: ["ubuntu-latest", "macos-latest"]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
         with:
-          python-version: 3.10
+          fetch-depth: 0
 
-      - run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Dependencies
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry install --with dev
+
+      - name: Get version from poetry version --short
+        id: get_version
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: |
+          echo "::set-output name=version::$(poetry version --short)"
+
+      - name: Set binary name
+        id: get_artifact_name
+        run: |
+          echo "::set-output name=artifact_name::airbyte-ci-${{steps.get_version.outputs.version}}-${{ matrix.os }}"
+
+      - name: Build Release
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry run poe build-release-binary ${{ steps.get_artifact_name.outputs.artifact_name}}
 
-      - run: cd airbyte-ci/pipelines/airbyte_ci
-      - run: poetry install --with dev
-      - run: poetry run pyinstaller --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name airbyte-ci-${{ matrix.os }} --onefile pipelines/cli/airbyte_ci.py
       - uses: actions/upload-artifact@v2
         with:
-          path: dist/*
+          name: ${{ steps.get_artifact_name.outputs.artifact_name}}
+          path: airbyte-ci/connectors/pipelines/dist/${{ steps.get_artifact_name.outputs.artifact_name}}
+
+      - id: "auth"
+        name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}"
+
+      - name: Upload Release to GCS
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ steps.get_artifact_name.outputs.artifact_name}}
+          destination: dev-airbyte-cloud-connector-metadata-service/airbyte-ci/releases/${{ steps.get_artifact_name.outputs.artifact_name}}
+          headers: |-
+            cache-control:public, max-age=10
+
+      # if master, upload as latest
+
+      - name: Set latest binary name
+        id: get_latest_artifact_name
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "::set-output name=artifact_name::airbyte-ci-${{ matrix.os }}"
+
+      - name: Upload Release to GCS
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ steps.get_artifact_name.outputs.artifact_name}}
+          destination: dev-airbyte-cloud-connector-metadata-service/airbyte-ci/releases/${{ steps.get_latest_artifact_name.outputs.artifact_name}}
+          headers: |-
+            cache-control:public, max-age=10

airbyte-ci/.gitignore

@@ -30,7 +30,6 @@ MANIFEST
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
 
 # Installer logs
 pip-log.txt

airbyte-ci/connectors/pipelines/README.md

@@ -123,7 +123,7 @@ At this point you can run `airbyte-ci` commands.
 
 | Option                                  | Default value                   | Mapped environment variable   | Description                                                                                 |
 | --------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
-| `--no-tui`                              |                                 |                               | Disables the Dagger terminal UI.                                                            |
+| `--enable-dagger-run/--disable-dagger-run` |  `--enable-dagger-run``      |                               | Disables the Dagger terminal UI.                                                            |
 | `--is-local/--is-ci`                    | `--is-local`                    |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
 | `--git-branch`                          | The checked out git branch name | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
 | `--git-revision`                        | The current branch head         | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
@@ -408,6 +408,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 2.6.0   | [#31930](https://github.com/airbytehq/airbyte/pull/31930)  | Merge airbyte-ci-internal into airbyte-ci                                                                 |
 | 2.5.8   | [#32402](https://github.com/airbytehq/airbyte/pull/32402)  | Set Dagger Cloud token for airbyters only                                                                 |
 | 2.5.7   | [#31628](https://github.com/airbytehq/airbyte/pull/31628)  | Add ClickPipelineContext class                                                                            |
 | 2.5.6   | [#32139](https://github.com/airbytehq/airbyte/pull/32139)  | Test coverage report on Python connector UnitTest.                                                        |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py

@@ -7,9 +7,9 @@
 
 import semver
 from dagger import Container
-from pipelines import consts
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
+from pipelines.helpers import git
 from pipelines.helpers.connectors import metadata_change_helpers
 from pipelines.models.steps import Step, StepResult, StepStatus
 
@@ -171,6 +171,6 @@ async def run_connector_version_bump_pipeline(
             add_changelog_entry_result = await add_changelog_entry.run()
             steps_results.append(add_changelog_entry_result)
             final_repo_dir = add_changelog_entry_result.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
             context.report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS")
     return context.report

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py

@@ -10,10 +10,10 @@
 from connector_ops.utils import ConnectorLanguage
 from dagger import Directory
 from jinja2 import Template
-from pipelines import consts
 from pipelines.airbyte_ci.connectors.bump_version.pipeline import AddChangelogEntry, BumpDockerImageTagInMetadata, get_bumped_version
 from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
+from pipelines.helpers import git
 from pipelines.helpers.connectors import metadata_change_helpers
 from pipelines.models.steps import Step, StepResult, StepStatus
 
@@ -266,7 +266,7 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s
             update_base_image_in_metadata_result = await update_base_image_in_metadata.run()
             steps_results.append(update_base_image_in_metadata_result)
             final_repo_dir = update_base_image_in_metadata_result.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
             context.report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS")
     return context.report
 
@@ -337,7 +337,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte
 
             # EXPORT MODIFIED FILES BACK TO HOST
             final_repo_dir = add_build_instructions_to_doc_results.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
 
             context.report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS")
     return context.report

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -8,6 +8,7 @@
 import logging
 import multiprocessing
 import os
+import sys
 from pathlib import Path
 from typing import List, Optional
 
@@ -19,7 +20,7 @@
 from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.cli.lazy_group import LazyGroup
 from pipelines.cli.telemetry import click_track_command
-from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext
+from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, LOCAL_PIPELINE_PACKAGE_PATH, CIContext
 from pipelines.helpers import github
 from pipelines.helpers.git import (
     get_current_git_branch,
@@ -56,7 +57,7 @@ def display_welcome_message() -> None:
     )
 
 
-def check_up_to_date() -> bool:
+def check_up_to_date(throw_as_error=False) -> bool:
     """Check if the installed version of pipelines is up to date."""
     latest_version = get_latest_version()
     if latest_version != __installed_version__:
@@ -68,7 +69,12 @@ def check_up_to_date() -> bool:
 
         🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
         """
-        raise Exception(upgrade_error_message)
+
+        if throw_as_error:
+            raise Exception(upgrade_error_message)
+        else:
+            logging.warning(upgrade_error_message)
+            return False
 
     main_logger.info(f"pipelines is up to date. Installed version: {__installed_version__}. Latest version: {latest_version}")
     return True
@@ -206,6 +212,38 @@ def check_local_docker_configuration():
         )
 
 
+def is_dagger_run_enabled_by_default() -> bool:
+    dagger_run_by_default = [
+        ["connectors", "test"],
+        ["connectors", "build"],
+        ["test"],
+        ["metadata_service"],
+    ]
+
+    for command_tokens in dagger_run_by_default:
+        if all(token in sys.argv for token in command_tokens):
+            return True
+
+    return False
+
+
+def check_dagger_wrap():
+    """
+    Check if the command is already wrapped by dagger run.
+    This is useful to avoid infinite recursion when calling dagger run from dagger run.
+    """
+    return os.getenv(DAGGER_WRAP_ENV_VAR_NAME) == "true"
+
+
+def is_current_process_wrapped_by_dagger_run() -> bool:
+    """
+    Check if the current process is wrapped by dagger run.
+    """
+    called_with_dagger_run = check_dagger_wrap()
+    main_logger.info(f"Called with dagger run: {called_with_dagger_run}")
+    return called_with_dagger_run
+
+
 async def get_modified_files_str(ctx: click.Context):
     modified_files = await get_modified_files(
         ctx.obj["git_branch"],
@@ -231,6 +269,7 @@ async def get_modified_files_str(ctx: click.Context):
     },
 )
 @click.version_option(__installed_version__)
+@click.option("--enable-dagger-run/--disable-dagger-run", default=is_dagger_run_enabled_by_default)
 @click.option("--is-local/--is-ci", default=True)
 @click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH")
 @click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION")
@@ -247,6 +286,7 @@ async def get_modified_files_str(ctx: click.Context):
 @click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str)
 @click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str)
 @click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str)
+@click.option("--ci-artifact-bucket-name", envvar="CI_ARTIFACT_BUCKET_NAME", type=str)
 @click.option(
     "--ci-gcs-credentials",
     help="The service account to use during CI.",
@@ -268,12 +308,20 @@ async def get_modified_files_str(ctx: click.Context):
 @click_ignore_unused_kwargs
 async def airbyte_ci(ctx: click.Context):  # noqa D103
     display_welcome_message()
+
+    if ctx.obj["enable_dagger_run"] and not is_current_process_wrapped_by_dagger_run():
+        main_logger.debug("Re-Running airbyte-ci with dagger run.")
+        from pipelines.cli.dagger_run import call_current_command_with_dagger_run
+
+        call_current_command_with_dagger_run()
+        return
+
     if ctx.obj["is_local"]:
         # This check is meaningful only when running locally
         # In our CI the docker host used by the Dagger Engine is different from the one used by the runner.
         check_local_docker_configuration()
 
-    check_up_to_date()
+    check_up_to_date(throw_as_error=ctx.obj["is_local"])
 
     if not ctx.obj["is_local"]:
         log_git_info(ctx)

airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py

@@ -14,7 +14,7 @@
 
 import pkg_resources
 import requests
-from pipelines.cli.airbyte_ci import set_working_directory_to_root
+from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME
 
 LOGGER = logging.getLogger(__name__)
 BIN_DIR = Path.home() / "bin"
@@ -72,6 +72,10 @@ def get_dagger_cli_version(dagger_path: Optional[str]) -> Optional[str]:
 
 
 def check_dagger_cli_install() -> str:
+    """
+    If the dagger CLI is not installed, install it.
+    """
+
     expected_dagger_cli_version = get_current_dagger_sdk_version()
     dagger_path = get_dagger_path()
     if dagger_path is None:
@@ -89,17 +93,21 @@ def check_dagger_cli_install() -> str:
     return dagger_path
 
 
-def main():
-    set_working_directory_to_root()
+def mark_dagger_wrap():
+    """
+    Mark that the dagger wrap has been applied.
+    """
+    os.environ[DAGGER_WRAP_ENV_VAR_NAME] = "true"
+
+
+def call_current_command_with_dagger_run():
+    mark_dagger_wrap()
     if (os.environ.get("AIRBYTE_ROLE") == "airbyter") or (os.environ.get("CI") == "True"):
         os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1]
 
     exit_code = 0
-    if len(sys.argv) > 1 and any([arg in ARGS_DISABLING_TUI for arg in sys.argv]):
-        command = ["airbyte-ci-internal"] + [arg for arg in sys.argv[1:] if arg != "--no-tui"]
-    else:
-        dagger_path = check_dagger_cli_install()
-        command = [dagger_path, "run", "airbyte-ci-internal"] + sys.argv[1:]
+    dagger_path = check_dagger_cli_install()
+    command = [dagger_path, "run"] + sys.argv
     try:
         try:
             LOGGER.info(f"Running command: {command}")
@@ -110,7 +118,3 @@ def main():
     except subprocess.CalledProcessError as e:
         exit_code = e.returncode
     sys.exit(exit_code)
-
-
-if __name__ == "__main__":
-    main()

airbyte-ci/connectors/pipelines/pipelines/consts.py

@@ -5,7 +5,6 @@
 import platform
 from enum import Enum
 
-import git
 from dagger import Platform
 
 PYPROJECT_TOML_FILE_PATH = "pyproject.toml"
@@ -29,6 +28,7 @@
     "x86_64": Platform("linux/amd64"),
     "aarch64": Platform("linux/amd64"),
     "arm64": Platform("linux/arm64"),
+    "aarch64": Platform("linux/amd64"),
     "amd64": Platform("linux/amd64"),
 }
 LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()]
@@ -47,8 +47,6 @@
 DOCKER_HOST_PORT = 2375
 DOCKER_TMP_VOLUME_NAME = "shared-tmp"
 DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache"
-REPO = git.Repo(search_parent_directories=True)
-REPO_PATH = REPO.working_tree_dir
 STATIC_REPORT_PREFIX = "airbyte-ci"
 PIP_CACHE_VOLUME_NAME = "pip_cache"
 PIP_CACHE_PATH = "/root/.cache/pip"
@@ -82,3 +80,6 @@ class INTERNAL_TOOL_PATHS(str, Enum):
     CI_CREDENTIALS = "airbyte-ci/connectors/ci_credentials"
     CONNECTOR_OPS = "airbyte-ci/connectors/connector_ops"
     METADATA_SERVICE = "airbyte-ci/connectors/metadata_service/lib"
+
+
+DAGGER_WRAP_ENV_VAR_NAME = "_DAGGER_WRAP_APPLIED"