introduce new github workflow for python connectors

Begin to factor out common setup steps into reuseable composite actions.

Add a common estuary-cdk Dockerfile

.github/actions/deploy/action.yaml

@@ -0,0 +1,66 @@
+name: Common Deployment Steps
+description: Implements common deployment steps of all connector workflows
+inputs:
+ connector:
+ description: Image name of this connector.
+ required: true
+ tag_version:
+ description: Version tag of the connector, such as `v1`.
+ required: true
+ tag_sha:
+ description: Short tag of the commit SHA1, such as 'f32dc1a'.
+ required: true
+ variants:
+ description: Space-separated variant image names of this connector.
+ default: ''
+ required: false
+
+runs:
+ using: "composite"
+ steps:
+
+ - name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag
+ shell: bash
+ run: |
+ for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+ echo "Building and pushing ${VARIANT}:${{ inputs.tag }}...";
+ docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \
+ --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \
+ --tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \
+ --file connector-variant.Dockerfile .;
+ docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }};
+ done
+
+ - name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags
+ if: ${{ github.event_name == 'push' }}
+ shell: bash
+ run: |
+ for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+ docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev;
+ docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+
+ docker image push ghcr.io/estuary/${VARIANT}:dev;
+ docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+ done
+
+ - name: Install psql
+ if: ${{ github.event_name == 'push' }}
+ shell: bash
+ run: sudo apt install postgresql
+
+ - name: Refresh connector tags for ${{ inputs.connector }}
+ if: ${{ github.event_name == 'push' }}
+ shell: bash
+ env:
+ PGHOST: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }}
+ PGUSER: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }}
+ PGPASSWORD: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }}
+ PGDATABASE: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }}
+ run: |
+ for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+ echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}'
+ WHERE connector_id IN (
+ SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}'
+ ) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql;
+ done
+

.github/actions/setup/action.yaml

@@ -0,0 +1,56 @@
+name: Common Setup Steps
+description: Implements common setup steps of all connector workflows
+
+inputs:
+ setup_gcloud:
+ description: Whether to set up a gcloud service account.
+ required: false
+ default: 'false'
+
+outputs:
+ tag_sha:
+ description: Short tag of the commit SHA1
+ value: ${{ steps.determine-sha-tag.outputs.tag }}
+
+runs:
+ using: "composite"
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Determine current SHA tag.
+ shell: bash
+ id: determine-sha-tag
+ run: |
+ TAG=$(echo $GITHUB_SHA | head -c7)
+ echo ::set-output name=tag::${TAG}
+
+ - name: Download latest Flow release binaries and add them to $PATH
+ shell: bash
+ run: |
+ ./fetch-flow.sh
+ echo "${PWD}/flow-bin" >> $GITHUB_PATH
+ 
+ - name: Login to GitHub package docker registry
+ shell: bash
+ run: |
+ echo "${{ secrets.GITHUB_TOKEN }}" | \
+ docker login --username ${{ github.actor }} --password-stdin ghcr.io
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ with:
+ driver-opts: network=host
+
+ - name: Create docker network flow-test
+ shell: bash
+ run: docker network create flow-test
+
+ - name: Set up Cloud SDK
+ if: ${{ inputs.setup_gcloud_sa }}
+ uses: google-github-actions/setup-gcloud@v0
+ with:
+ project_id: ${{ secrets.GCP_PROJECT_ID }}
+ service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+ export_default_credentials: true

.github/workflows/ci.yaml

@@ -7,9 +7,9 @@ concurrency:
 on:
  push:
  # TODO: Remove jshearer/python_connector_testing before merging
- branches: [main, jshearer/python_connector_testing]
+ branches: [nothing]
  pull_request:
- branches: [main]
+ branches: [nothing]
 
 jobs:
  build_base_image:
@@ -121,9 +121,6 @@ jobs:
  - connector: source-shopify
  connector_type: capture
  python: true
- - connector: source-asana
- connector_type: capture
- python: true
  - connector: source-airtable
  connector_type: capture
  python: true

.github/workflows/python-new.yaml

@@ -0,0 +1,79 @@
+name: Python Connectors
+
+on:
+ push:
+ branches: [main]
+ paths:
+ - "estuary-cdk/**"
+ - "source-asana/**"
+ - "source-google-sheets-native/**"
+ - "source-hubspot-native/**"
+ pull_request:
+ branches: [main]
+ paths:
+ - "estuary-cdk/**"
+ - "source-asana/**"
+ - "source-google-sheets-native/**"
+ - "source-hubspot-native/**"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ py_connector:
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ connector:
+ - name: source-asana
+ type: capture
+ version: v1
+ - name: source-hubspot-native
+ type: capture
+ version: v1
+ - name: source-google-sheets-native
+ type: capture
+ version: v1
+
+ steps:
+ - id: setup
+ uses: .github/actions/setup/action.yaml
+ with:
+ setup_gcloud: false
+
+ - name: Setup Python for ${{ matrix.connector }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install Poetry
+ uses: snok/install-poetry@v1
+
+ - name: Python connector ${{ matrix.connector }} tests
+ run: |
+ cd ${{ matrix.connector }}
+ poetry install
+ source $(poetry env info --path)/bin/activate
+ cd ..
+ pytest .
+
+ - name: Build ${{ matrix.connector }} Python Docker Image
+ uses: docker/build-push-action@v2
+ with:
+ context: .
+ file: estuary-cdk/common.Dockerfile
+ load: true
+ build-args: |
+ CONNECTOR_NAME=${{ matrix.connector.name }}
+ CONNECTOR_TYPE=${{ matrix.connector.type }}
+ tags: ghcr.io/estuary/${{ matrix.connector }}:local
+
+ - id: deploy
+ uses: .github/actions/deploy/action.yaml
+ with:
+ connector: ${{ matrix.connector.name }}
+ tag_sha: ${{ steps.setup.outputs.tag_sha }}
+ tag_version: ${{ matrix.connector.version }}
+ variants: ${{ matrix.connector.variants }}

estuary-cdk/common.Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+FROM python:3.12-slim as base
+FROM base as builder
+
+ARG CONNECTOR_NAME
+
+RUN apt-get update && \
+ apt install -y --no-install-recommends \
+ python3-poetry 
+
+RUN python -m venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv
+
+WORKDIR /opt/${CONNECTOR_NAME}
+COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME}
+COPY estuary-cdk /opt/estuary-cdk
+
+RUN poetry install
+
+
+FROM base as runner
+
+ARG CONNECTOR_NAME
+ARG CONNECTOR_TYPE
+ARG DOCS_URL
+
+LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE}
+LABEL FLOW_RUNTIME_CODEC=json
+
+COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME
+COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk
+COPY --from=builder /opt/venv /opt/venv
+
+ENV DOCS_URL=${DOCS_URL}
+ENV CONNECTOR_NAME=${CONNECTOR_NAME}
+
+CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_')