estuary-cdk and new connectors

.github/actions/deploy/action.yaml

@@ -0,0 +1,78 @@
+name: Common Deployment Steps
+description: Implements common deployment steps of all connector workflows
+inputs:
+  connector:
+    description: Image name of this connector.
+    required: true
+  pg_database:
+    description: Postgres database to use.
+    required: true
+  pg_host:
+    description: Postgres host to use.
+    required: true
+  pg_password:
+    description: Postgres password to use.
+    required: true
+  pg_user:
+    description: Postgres user to use.
+    required: true
+  tag_sha:
+    description: Short tag of the commit SHA1, such as 'f32dc1a'.
+    required: true
+  tag_version:
+    description: Version tag of the connector, such as `v1`.
+    required: true
+  variants:
+    description: Space-separated variant image names of this connector.
+    default: ''
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Push ${{ inputs.connector }} ${{ inputs.variants }} image(s) with commit SHA tag
+      shell: bash
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          echo "Building and pushing ${VARIANT}:${{ inputs.tag }}...";
+          docker build --build-arg BASE_CONNECTOR=ghcr.io/estuary/${{ inputs.connector }}:local \
+                       --build-arg DOCS_URL=https://go.estuary.dev/${VARIANT} \
+                       --tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} \
+                       --file connector-variant.Dockerfile .;
+          docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }};
+        done
+
+    - name: Push ${{ inputs.connector }} image(s) with 'dev' and '${{ inputs.tag_version }}' tags
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:dev;
+          docker image tag ghcr.io/estuary/${VARIANT}:${{ inputs.tag_sha }} ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+
+          docker image push ghcr.io/estuary/${VARIANT}:dev;
+          docker image push ghcr.io/estuary/${VARIANT}:${{ inputs.tag_version }};
+        done
+
+    - name: Install psql
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: sudo apt install postgresql
+
+    - name: Refresh connector tags for ${{ inputs.connector }}
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      env:
+        PGDATABASE: ${{ inputs.pg_database }}
+        PGHOST: ${{ inputs.pg_host }}
+        PGPASSWORD: ${{ inputs.pg_password }}
+        PGUSER: ${{ inputs.pg_user }}
+      run: |
+        for VARIANT in ${{ inputs.connector }} ${{ inputs.variants }}; do
+          echo "UPDATE connector_tags SET job_status='{\"type\": \"queued\"}'
+                  WHERE connector_id IN (
+                    SELECT id FROM connectors WHERE image_name='ghcr.io/estuary/${VARIANT}'
+                  ) AND image_tag IN (':${{ inputs.tag_version }}', ':dev');" | psql;
+        done
+

.github/actions/setup/action.yaml

@@ -0,0 +1,58 @@
+name: Common Setup Steps
+description: Implements common setup steps of all connector workflows
+
+inputs:
+  github_token:
+    description: GitHub token to use.
+    required: true
+  gcp_project_id:
+    description: GCP Project for which to set up a service account.
+    required: true
+  gcp_service_account_key:
+    description: GCP Service account.
+    required: true
+
+outputs:
+  tag_sha:
+    description: Short tag of the commit SHA1
+    value: ${{ steps.determine-sha-tag.outputs.tag }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Determine current SHA tag.
+      shell: bash
+      id: determine-sha-tag
+      run: |
+        TAG=$(echo $GITHUB_SHA | head -c7)
+        echo "tag=${TAG}" >> $GITHUB_OUTPUT
+
+    - name: Download latest Flow release binaries and add them to $PATH
+      shell: bash
+      run: |
+        ./fetch-flow.sh
+        echo "${PWD}/flow-bin" >> $GITHUB_PATH
+
+    - name: Login to GitHub package docker registry
+      shell: bash
+      run: |
+        echo "${{ inputs.github_token }}" | \
+          docker login --username ${{ github.actor }} --password-stdin ghcr.io
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
+
+    - name: Create docker network flow-test
+      shell: bash
+      run: docker network create flow-test
+
+    - name: Authenticate with GCloud
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ inputs.gcp_service_account_key }}
+
+    - name: Set up GCloud SDK
+      uses: google-github-actions/setup-gcloud@v2

.github/workflows/ci.yaml

@@ -6,16 +6,13 @@ concurrency:
 
 on:
   push:
-    # TODO: Remove jshearer/python_connector_testing before merging
-    branches: [main, jshearer/python_connector_testing]
+    branches: [main]
   pull_request:
     branches: [main]
 
 jobs:
   build_base_image:
     runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
     steps:
       - uses: actions/checkout@v2
         with:
@@ -121,9 +118,6 @@ jobs:
           - connector: source-shopify
             connector_type: capture
             python: true
-          - connector: source-asana
-            connector_type: capture
-            python: true
           - connector: source-airtable
             connector_type: capture
             python: true

.github/workflows/python.yaml

@@ -0,0 +1,88 @@
+name: Python Connectors
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "estuary-cdk/**"
+      - "source-asana/**"
+      - "source-google-sheets-native/**"
+      - "source-hubspot-native/**"
+  pull_request:
+    branches: [main]
+    paths:
+      - "estuary-cdk/**"
+      - "source-asana/**"
+      - "source-google-sheets-native/**"
+      - "source-hubspot-native/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  py_connector:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        connector:
+          - name: source-asana
+            type: capture
+            version: v1
+          - name: source-hubspot-native
+            type: capture
+            version: v1
+          - name: source-google-sheets-native
+            type: capture
+            version: v1
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Common Setup
+        id: setup
+        uses: ./.github/actions/setup
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          gcp_project_id: ${{ secrets.GCP_PROJECT_ID }}
+          gcp_service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: ${{ matrix.connector.name }} tests
+        run: |
+          cd ${{ matrix.connector.name }}
+          poetry install
+          source $(poetry env info --path)/bin/activate
+          cd ..
+          pytest ${{ matrix.connector.name }}
+
+      - name: Build ${{ matrix.connector.name }} Docker Image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: estuary-cdk/common.Dockerfile
+          load: true
+          build-args: |
+            CONNECTOR_NAME=${{ matrix.connector.name }}
+            CONNECTOR_TYPE=${{ matrix.connector.type }}
+          tags: ghcr.io/estuary/${{ matrix.connector.name }}:local
+
+      - name: Deployment
+        uses: ./.github/actions/deploy
+        with:
+          connector: ${{ matrix.connector.name }}
+          pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }}
+          pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }}
+          pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }}
+          pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }}
+          tag_sha: ${{ steps.setup.outputs.tag_sha }}
+          tag_version: ${{ matrix.connector.version }}
+          variants: ${{ matrix.connector.variants }}

.vscode/settings.json

@@ -1,12 +1,10 @@
 {
-    "python.linting.enabled": true,
-    "python.linting.mypyEnabled": true,
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter"
     },
-    "python.formatting.provider": "none",
     "files.exclude": {
         "**/*.pyc": {"when": "$(basename).py"}, 
+        "**/.mypy_cache": true,
         "**/.pytest_cache": true,
         "**/__pycache__": true,
     },

estuary-cdk/common.Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+FROM python:3.12-slim as base
+FROM base as builder
+
+ARG CONNECTOR_NAME
+
+RUN apt-get update && \
+    apt install -y --no-install-recommends \
+    python3-poetry 
+
+RUN python -m venv /opt/venv
+ENV VIRTUAL_ENV=/opt/venv
+
+WORKDIR /opt/${CONNECTOR_NAME}
+COPY ${CONNECTOR_NAME} /opt/${CONNECTOR_NAME}
+COPY estuary-cdk /opt/estuary-cdk
+
+RUN poetry install
+
+
+FROM base as runner
+
+ARG CONNECTOR_NAME
+ARG CONNECTOR_TYPE
+ARG DOCS_URL
+
+LABEL FLOW_RUNTIME_PROTOCOL=${CONNECTOR_TYPE}
+LABEL FLOW_RUNTIME_CODEC=json
+
+COPY --from=builder /opt/$CONNECTOR_NAME /opt/$CONNECTOR_NAME
+COPY --from=builder /opt/estuary-cdk /opt/estuary-cdk
+COPY --from=builder /opt/venv /opt/venv
+
+ENV DOCS_URL=${DOCS_URL}
+ENV CONNECTOR_NAME=${CONNECTOR_NAME}
+
+CMD /opt/venv/bin/python -m $(echo "$CONNECTOR_NAME" | tr '-' '_')