Airbyte-ci: Generate binary files for airbyte-ci

.github/actions/run-dagger-pipeline/action.yml

@@ -104,7 +104,7 @@ runs:
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"

.github/workflows/airbyte-ci-release-experiment.yml

@@ -1,32 +1,125 @@
-name: Connector Ops CI - Experimental Airbyte CI Release
-
-# Note this is a workflow simply to test out if github can build using macos
+# TODO remove -experiment from the workflow file name after merging
+name: Connector Ops CI - Airbyte CI Release
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
+  push:
+    branches:
+      - master
+    paths:
+      - "airbyte-ci/connectors/pipelines/**"
   workflow_dispatch:
+
+env:
+  DEV_GCS_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service
+  PROD_GCS_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service
+  BINARY_FILE_NAME: airbyte-ci
+
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest"]
+        os: ["ubuntu", "macos"]
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - name: Checkout Airbyte
+        id: checkout_airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.sha }} # This is required to make sure that the same commit is checked out on all runners
+
+      - name: Get short SHA
+        id: get_short_sha
+        uses: benjlevesque/short-sha@v2.2
+
+      - name: Install Python
+        id: install_python
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.10
+          python-version: "3.10"
 
-      - run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+      - name: Install Poetry
+        id: install_poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Dependencies
+        id: install_dependencies
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry install --with dev
+
+      - name: Build release
+        id: build_release
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }}
 
-      - run: cd airbyte-ci/pipelines/airbyte_ci
-      - run: poetry install --with dev
-      - run: poetry run pyinstaller --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name airbyte-ci-${{ matrix.os }} --onefile pipelines/cli/airbyte_ci.py
       - uses: actions/upload-artifact@v2
         with:
-          path: dist/*
+          name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }}
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}"
+
+      - name: Upload pre-release to GCS dev bucket
+        id: upload_pre_release_to_gcs
+        if: github.ref != 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print pre-release public url
+        id: print_pre_release_public_url
+        run: |
+          echo "https://storage.googleapis.com/${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}/${{ env.BINARY_FILE_NAME }}"
+
+      # if master, upload per version and latest to prod bucket
+
+      - name: Set version from poetry version --short
+        id: set_version
+        if: github.ref == 'refs/heads/master'
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: |
+          echo "::set-output name=version::$(poetry version --short)"
+
+      - name: Upload version release to GCS prod bucket
+        id: upload_version_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print release version public url
+        id: print_version_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}/${{ env.BINARY_FILE_NAME }}"
+
+      - name: Upload latest release to GCS prod bucket
+        id: upload_latest_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print latest release public url
+        id: print_latest_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest/${{ env.BINARY_FILE_NAME }}"

airbyte-ci/connectors/pipelines/README.md

@@ -124,17 +124,17 @@ At this point you can run `airbyte-ci` commands.
 
 #### Options
 
-| Option                                  | Default value                   | Mapped environment variable   | Description                                                                                 |
-| --------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
-| `--no-tui`                              |                                 |                               | Disables the Dagger terminal UI.                                                            |
-| `--is-local/--is-ci`                    | `--is-local`                    |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
-| `--git-branch`                          | The checked out git branch name | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
-| `--git-revision`                        | The current branch head         | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
-| `--diffed-branch`                       | `origin/master`                 |                               | Branch to which the git diff will happen to detect new or modified files.                   |
-| `--gha-workflow-run-id`                 |                                 |                               | GHA CI only - The run id of the GitHub action workflow                                      |
-| `--ci-context`                          | `manual`                        |                               | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
-| `--pipeline-start-timestamp`            | Current epoch time              | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation.       |
-| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs`            |                               | Flag to show or hide the dagger logs.                                                       |
+| Option                                     | Default value                                                                                  | Mapped environment variable   | Description                                                                                 |
+| ------------------------------------------ | ---------------------------------------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
+| `--enable-dagger-run/--disable-dagger-run` | `--enable-dagger-run``      |                               | Disables the Dagger terminal UI. |                               |                                                                                             |
+| `--is-local/--is-ci`                       | `--is-local`                                                                                   |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
+| `--git-branch`                             | The checked out git branch name                                                                | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
+| `--git-revision`                           | The current branch head                                                                        | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
+| `--diffed-branch`                          | `origin/master`                                                                                |                               | Branch to which the git diff will happen to detect new or modified files.                   |
+| `--gha-workflow-run-id`                    |                                                                                                |                               | GHA CI only - The run id of the GitHub action workflow                                      |
+| `--ci-context`                             | `manual`                                                                                       |                               | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
+| `--pipeline-start-timestamp`               | Current epoch time                                                                             | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation.       |
+| `--show-dagger-logs/--hide-dagger-logs`    | `--hide-dagger-logs`                                                                           |                               | Flag to show or hide the dagger logs.                                                       |
 
 ### <a id="connectors-command-subgroup"></a>`connectors` command subgroup
 
@@ -433,6 +433,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 2.7.0   | [#31930](https://github.com/airbytehq/airbyte/pull/31930)  | Merge airbyte-ci-internal into airbyte-ci                                                                 |
 | 2.6.0   | [#31831](https://github.com/airbytehq/airbyte/pull/31831)  | Add `airbyte-ci format` commands, remove connector-specific formatting check                              |
 | 2.5.9   | [#32427](https://github.com/airbytehq/airbyte/pull/32427)  | Re-enable caching for source-postgres                                                                     |
 | 2.5.8   | [#32402](https://github.com/airbytehq/airbyte/pull/32402)  | Set Dagger Cloud token for airbyters only                                                                 |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py

@@ -7,9 +7,9 @@
 
 import semver
 from dagger import Container
-from pipelines import consts
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
+from pipelines.helpers import git
 from pipelines.helpers.connectors import metadata_change_helpers
 from pipelines.models.steps import Step, StepResult, StepStatus
 
@@ -171,6 +171,6 @@ async def run_connector_version_bump_pipeline(
             add_changelog_entry_result = await add_changelog_entry.run()
             steps_results.append(add_changelog_entry_result)
             final_repo_dir = add_changelog_entry_result.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
             context.report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS")
     return context.report

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py

@@ -10,10 +10,10 @@
 from connector_ops.utils import ConnectorLanguage
 from dagger import Directory
 from jinja2 import Template
-from pipelines import consts
 from pipelines.airbyte_ci.connectors.bump_version.pipeline import AddChangelogEntry, BumpDockerImageTagInMetadata, get_bumped_version
 from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
+from pipelines.helpers import git
 from pipelines.helpers.connectors import metadata_change_helpers
 from pipelines.models.steps import Step, StepResult, StepStatus
 
@@ -266,7 +266,7 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s
             update_base_image_in_metadata_result = await update_base_image_in_metadata.run()
             steps_results.append(update_base_image_in_metadata_result)
             final_repo_dir = update_base_image_in_metadata_result.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
             context.report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS")
     return context.report
 
@@ -337,7 +337,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte
 
             # EXPORT MODIFIED FILES BACK TO HOST
             final_repo_dir = add_build_instructions_to_doc_results.output_artifact
-            await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH))
+            await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
 
             context.report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS")
     return context.report

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -8,6 +8,7 @@
 import logging
 import multiprocessing
 import os
+import sys
 from pathlib import Path
 from typing import List, Optional
 
@@ -19,7 +20,7 @@
 from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.cli.lazy_group import LazyGroup
 from pipelines.cli.telemetry import click_track_command
-from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext
+from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, LOCAL_PIPELINE_PACKAGE_PATH, CIContext
 from pipelines.helpers import github
 from pipelines.helpers.git import (
     get_current_git_branch,
@@ -56,7 +57,7 @@ def display_welcome_message() -> None:
     )
 
 
-def check_up_to_date() -> bool:
+def check_up_to_date(throw_as_error=False) -> bool:
     """Check if the installed version of pipelines is up to date."""
     latest_version = get_latest_version()
     if latest_version != __installed_version__:
@@ -68,7 +69,12 @@ def check_up_to_date() -> bool:
 
         🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
         """
-        raise Exception(upgrade_error_message)
+
+        if throw_as_error:
+            raise Exception(upgrade_error_message)
+        else:
+            logging.warning(upgrade_error_message)
+            return False
 
     main_logger.info(f"pipelines is up to date. Installed version: {__installed_version__}. Latest version: {latest_version}")
     return True
@@ -206,6 +212,38 @@ def check_local_docker_configuration():
         )
 
 
+def is_dagger_run_enabled_by_default() -> bool:
+    dagger_run_by_default = [
+        ["connectors", "test"],
+        ["connectors", "build"],
+        ["test"],
+        ["metadata_service"],
+    ]
+
+    for command_tokens in dagger_run_by_default:
+        if all(token in sys.argv for token in command_tokens):
+            return True
+
+    return False
+
+
+def check_dagger_wrap():
+    """
+    Check if the command is already wrapped by dagger run.
+    This is useful to avoid infinite recursion when calling dagger run from dagger run.
+    """
+    return os.getenv(DAGGER_WRAP_ENV_VAR_NAME) == "true"
+
+
+def is_current_process_wrapped_by_dagger_run() -> bool:
+    """
+    Check if the current process is wrapped by dagger run.
+    """
+    called_with_dagger_run = check_dagger_wrap()
+    main_logger.info(f"Called with dagger run: {called_with_dagger_run}")
+    return called_with_dagger_run
+
+
 async def get_modified_files_str(ctx: click.Context):
     modified_files = await get_modified_files(
         ctx.obj["git_branch"],
@@ -232,6 +270,7 @@ async def get_modified_files_str(ctx: click.Context):
     },
 )
 @click.version_option(__installed_version__)
+@click.option("--enable-dagger-run/--disable-dagger-run", default=is_dagger_run_enabled_by_default)
 @click.option("--is-local/--is-ci", default=True)
 @click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH")
 @click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION")
@@ -248,6 +287,7 @@ async def get_modified_files_str(ctx: click.Context):
 @click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str)
 @click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str)
 @click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str)
+@click.option("--ci-artifact-bucket-name", envvar="CI_ARTIFACT_BUCKET_NAME", type=str)
 @click.option(
     "--ci-gcs-credentials",
     help="The service account to use during CI.",
@@ -269,12 +309,20 @@ async def get_modified_files_str(ctx: click.Context):
 @click_ignore_unused_kwargs
 async def airbyte_ci(ctx: click.Context):  # noqa D103
     display_welcome_message()
+
+    if ctx.obj["enable_dagger_run"] and not is_current_process_wrapped_by_dagger_run():
+        main_logger.debug("Re-Running airbyte-ci with dagger run.")
+        from pipelines.cli.dagger_run import call_current_command_with_dagger_run
+
+        call_current_command_with_dagger_run()
+        return
+
     if ctx.obj["is_local"]:
         # This check is meaningful only when running locally
         # In our CI the docker host used by the Dagger Engine is different from the one used by the runner.
         check_local_docker_configuration()
 
-    check_up_to_date()
+    check_up_to_date(throw_as_error=ctx.obj["is_local"])
 
     if not ctx.obj["is_local"]:
         log_git_info(ctx)