Skip to content

✨ New: Snowflake Cortex Destination 🚀 #28407

✨ New: Snowflake Cortex Destination 🚀

✨ New: Snowflake Cortex Destination 🚀 #28407

name: Internal Poetry packages CI
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
workflow_dispatch:
inputs:
airbyte_ci_subcommand:
description: "Subcommand to pass to the 'airbyte-ci test' command"
default: "--poetry-package-path=airbyte-ci/connectors/pipelines"
pull_request:
types:
- opened
- reopened
- synchronize
jobs:
changes:
runs-on: ubuntu-latest
outputs:
internal_poetry_packages: ${{ steps.changes.outputs.internal_poetry_packages }}
steps:
- name: Checkout Airbyte
if: github.event_name != 'pull_request'
uses: actions/checkout@v4
- id: changes
uses: dorny/paths-filter@v2
with:
# Note: expressions within a filter are OR'ed
filters: |
# This list is duplicated in `pipelines/airbyte_ci/test/__init__.py`
internal_poetry_packages:
- airbyte-ci/connectors/pipelines/**
- airbyte-ci/connectors/base_images/**
- airbyte-ci/connectors/common_utils/**
- airbyte-ci/connectors/connector_ops/**
- airbyte-ci/connectors/connectors_qa/**
- airbyte-ci/connectors/ci_credentials/**
- airbyte-ci/connectors/metadata_service/lib/**
- airbyte-ci/connectors/metadata_service/orchestrator/**
- airbyte-cdk/python/**
- airbyte-integrations/bases/connector-acceptance-test/**
run-tests:
needs: changes
# We only run the Internal Poetry packages CI job if there are changes to the packages on a non-forked PR
if: needs.changes.outputs.internal_poetry_packages == 'true'
name: Internal Poetry packages CI
runs-on: tooling-test-large
permissions:
pull-requests: read
statuses: write
steps:
# The run-tests job will be triggered if a fork made changes to the internal poetry packages.
# We don't want forks to make changes to the internal poetry packages.
# So we fail the job if the PR is from a fork, it will make the required CI check fail.
- name: Check if PR is from a fork
id: check-if-pr-is-from-fork
if: github.event_name == 'pull_request'
shell: bash
run: |
if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
echo "PR is from a fork. Exiting workflow..."
exit 78
fi
- name: Checkout Airbyte
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.ref }}
- name: Extract branch name [WORKFLOW DISPATCH]
shell: bash
if: github.event_name == 'workflow_dispatch'
run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
id: extract_branch
- name: Fetch last commit id from remote branch [PULL REQUESTS]
if: github.event_name == 'pull_request'
id: fetch_last_commit_id_pr
run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
- name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
if: github.event_name == 'workflow_dispatch'
id: fetch_last_commit_id_wd
run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
- name: Run poe tasks for modified internal packages [PULL REQUEST]
if: github.event_name == 'pull_request'
id: run-airbyte-ci-test-pr
uses: ./.github/actions/run-airbyte-ci
with:
context: "pull_request"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
git_branch: ${{ github.head_ref }}
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
subcommand: "test --modified"
- name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH]
id: run-airbyte-ci-test-workflow-dispatch
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/run-airbyte-ci
with:
context: "manual"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
git_branch: ${{ steps.extract_branch.outputs.branch }}
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
subcommand: "test ${{ inputs.airbyte_ci_subcommand}}"