Merge branch 'master' of github.com:airbytehq/airbyte into bmoric/con…

…vert-health-micronaut

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.25
+current_version = 0.40.26
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.25
+VERSION=0.40.26
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/.gitignore

@@ -0,0 +1 @@
+connector_org_review_requirements.yaml

.github/actions/build-branch/action.yml

@@ -4,6 +4,10 @@ inputs:
   branch_version_tag:
     description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
     required: false
+  build_docker_images:
+    description: 'Build docker images'
+    default: 'true'
+    required: false
 outputs:
   branch_version_tag:
     description: "Tag used for jars and docker images. Either user specified or auto generated as `dev-<commit_hash>`"
@@ -20,35 +24,17 @@ runs:
         [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "branch_version_tag=${{ inputs.branch_version_tag }}" >> $GITHUB_OUTPUT \
           || { short_hash=$(git rev-parse --short=10 HEAD); echo "branch_version_tag=dev-$short_hash" >> $GITHUB_OUTPUT ; }
 
-    - uses: actions/setup-java@v3
-      with:
-        distribution: "zulu"
-        java-version: "17"
-
-    - uses: actions/setup-node@v3
-      with:
-        node-version: "lts/*"
-
-    - uses: actions/setup-python@v4
-      with:
-        python-version: "3.9"
+    - name: Prepare Runner for Building
+      uses: ./.github/actions/runner-prepare-for-build
 
-    - name: Set up CI Gradle Properties
-      run: |
-        mkdir -p ~/.gradle/
-        cat > ~/.gradle/gradle.properties <<EOF
-        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-        org.gradle.workers.max=8
-        org.gradle.vfs.watch=false
-        EOF
+    - name: Build with Docker Images
+      if: inputs.build_docker_images == 'true'
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
       shell: bash
 
-    - name: Build
-      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
+    - name: Build without Docker Images
+      if: inputs.build_docker_images != 'true'
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew test --scan
       shell: bash
 
     - name: Publish to Maven Local

.github/actions/runner-prepare-for-build/action.yml

@@ -0,0 +1,47 @@
+name: 'Prepare Runner for Building'
+description: 'Prepare Runner for Building project'
+inputs:
+  install_java:
+    description: ''
+    required: false
+    default: 'true'
+  install_node:
+    description: ''
+    required: false
+    default: 'true'
+  install_python:
+    description: ''
+    required: false
+    default: 'true'
+runs:
+  using: "composite"
+  steps:
+    - if: inputs.install_java == 'true'
+      uses: actions/setup-java@v3
+      with:
+        distribution: "zulu"
+        java-version: "17"
+
+    - if: inputs.install_node == 'true'
+      uses: actions/setup-node@v3
+      with:
+        node-version: "lts/*"
+
+    - if: inputs.install_python == 'true'
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.9"
+
+    - if: inputs.install_java == 'true'
+      name: Set up CI Gradle Properties
+      run: |
+        mkdir -p ~/.gradle/
+        cat > ~/.gradle/gradle.properties <<EOF
+        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        org.gradle.vfs.watch=false
+        EOF
+      shell: bash

.github/actions/start-aws-runner/action.yml

@@ -41,7 +41,7 @@ runs:
         aws-region: us-east-2
     - name: Start EC2 runner
       id: start-ec2-runner
-      uses: airbytehq/ec2-github-runner@base64v1.1.0
+      uses: airbytehq/ec2-github-runner@MSGv0.0.2
       with:
         mode: start
         github-token: ${{ inputs.github-token }}

.github/workflows/approve-and-merge-dispatch.yml

@@ -0,0 +1,36 @@
+name: Approve and Merge Command Dispatch
+on:
+  issue_comment:
+    types: [created]
+jobs:
+  approveAndMergeDispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OCTAVIA_GITHUB_RUNNER_TOKEN }} \
+            ${{ secrets.SUPERTOPHER_PAT }}
+
+      - name: Auto Approve Slash Command Dispatch
+        uses: peter-evans/slash-command-dispatch@v3
+        id: scd
+        with:
+          token: ${{ env.PAT }}
+          permission: write
+          issue-type: pull-request
+          repository: airbytehq/airbyte-cloud
+          dispatch-type: repository
+          commands: |
+            approve-and-merge
+
+      - name: Edit comment with error message
+        if: steps.scd.outputs.error-message
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          body: |
+            > Error: ${{ steps.scd.outputs.error-message }}

.github/workflows/connector_ops_ci.yml

@@ -1,9 +1,11 @@
 name: Connector Ops CI
 
 on:
+  workflow_dispatch:
   pull_request:
     paths:
       - "airbyte-integrations/connectors/source-**"
+  pull_request_review:
 jobs:
   test-strictness-level:
     name: "Check test strictness level"
@@ -21,3 +23,33 @@ jobs:
         run: pip install --quiet -e ./tools/ci_connector_ops
       - name: Check test strictness level
         run: check-test-strictness-level
+  check-review-requirements:
+    name: "Check if a review is required from Connector teams"
+    runs-on: ubuntu-latest
+
+    if: ${{ github.repository == 'airbytehq/airbyte' }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install ci-connector-ops package
+        run: pip install --quiet -e ./tools/ci_connector_ops
+      - name: Write review requirements file
+        id: write-review-requirements-file
+        run: write-review-requirements-file >> $GITHUB_OUTPUT
+      - name: Get mandatory reviewers
+        id: get-mandatory-reviewers
+        run: print-mandatory-reviewers >> $GITHUB_OUTPUT
+      - name: Check if the review requirements are met
+        if: steps.write-review-requirements-file.outputs.CREATED_REQUIREMENTS_FILE == 'true'
+        uses: Automattic/action-required-review@v3
+        with:
+          status: ${{ steps.get-mandatory-reviewers.outputs.MANDATORY_REVIEWERS  }}
+          token: ${{ secrets.OCTAVIA_4_ROOT_ACCESS }}
+          requirements-file: .github/connector_org_review_requirements.yaml
+

.github/workflows/deploy-docs-site.yml

@@ -7,13 +7,20 @@ on:
     paths:
       - "docs/**"
 
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+    paths:
+      - "docs/**"
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
-  dummy-job:
-    name: Deploy Docs Assets
-    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+  build-and-deploy-docs:
+    name: Build and Deploy Docs Assets
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository
@@ -29,22 +36,47 @@ jobs:
           cache: "yarn"
           cache-dependency-path: docusaurus
 
+
       - name: Run Docusaurus
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: ./tools/bin/deploy_docusaurus
+        run: |-
+          export SKIP_DEPLOY="yes"
+          if [ "${{github.event_name}}" = 'push' -o "${{github.event_name}}" = 'workflow_dispatch' ]; then
+              export SKIP_DEPLOY="no"
+          fi
+
+          ./tools/bin/deploy_docusaurus
+
+
+      - name: Notify Slack -- deploy failed
+        if: always() && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && job.status != 'success'
+        uses: abinoda/slack-action@master
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          # 'C02TYDSUM8F' => '#dev-deploys'
+          # 'C03BEADRPNY' => '#docs'
+          args: >-
+            {\"channel\":\"C03BEADRPNY\",\"attachments\":[
+            {\"color\":\"#ff0000\",\"blocks\":[
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs deploy fails on the latest master :bangbang:\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}*\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run>\"}}
+            ]}]}
 
-      - name: Notify Slack channel on failure
+      - name: Notify Slack -- deploy succeeded
+        if: always() && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && job.status == 'success'
         uses: abinoda/slack-action@master
-        if: failure()
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
         with:
-          # 'C032Y32T065' channel => '#docs'
+          # 'C02TYDSUM8F' => '#dev-deploys'
+          # 'C03BEADRPNY' => '#docs'
           args: >-
-            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
-            {\"type\":\"divider\"},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs build fails on the latest master :bangbang: \n\n\"}},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
-            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
-            {\"type\":\"divider\"}]}
+            {\"channel\":\"C03BEADRPNY\",\"attachments\":[
+            {\"color\":\"#00ff00\",\"blocks\":[
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs deploy was successful :tada:\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}*\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run>\"}}
+            ]}]}

.github/workflows/publish-oss-for-cloud.yml

@@ -1,6 +1,9 @@
 name: Publish OSS Artifacts for Cloud
-concurrency:
-  group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+
+env:
+  # enable gradle remote build cache
+  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
 
 on:
   workflow_dispatch:
@@ -64,6 +67,12 @@ jobs:
           cat $GITHUB_OUTPUT || true # for the sake of investigation
 
   oss-branch-build:
+    concurrency:
+      # only allow one workflow run at a time for a given SHA
+      # to prevent multiple runs from pushing artifacts for the same SHA at the same time
+      # note: using inputs in the group expression only works when specifying concurrency at the job level
+      group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+      cancel-in-progress: false
     name: "Gradle Build and Publish"
     needs:
       - start-runner
@@ -80,6 +89,7 @@ jobs:
         uses: ./.github/actions/build-branch
         with:
           branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }}
+          build_docker_images: 'false'
 
       - name: Publish Dev Jars
         env:
@@ -97,6 +107,12 @@ jobs:
         shell: bash
 
   docker-push:
+    concurrency:
+      # only allow one workflow run at a time for a given SHA
+      # to prevent multiple runs from pushing artifacts for the same SHA at the same time
+      # note: using inputs in the group expression only works when specifying concurrency at the job level
+      group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+      cancel-in-progress: false
     name: "Push Docker Images"
     needs:
       - start-runner
@@ -123,6 +139,15 @@ jobs:
           echo "GIT_REVISION=${GIT_REVISION}" >> $GITHUB_ENV
         shell: bash
 
+      - name: Prepare Runner for Building
+        uses: ./.github/actions/runner-prepare-for-build
+
+        # Put tars/artifacts in the correct build directories so docker buildx can find the artifacts it needs when building docker images
+      - name: Prepare Docker context
+        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew copyGeneratedTar airbyte-db:db-lib:copyInitSql
+        shell: bash
+
+        # Build docker images using docker buildx (for multi platform)
       - name: Push Docker Images
         env:
           VERSION: ${{ needs.generate-tags.outputs.dev_tag }}
@@ -133,3 +158,37 @@ jobs:
       - name: Cleanup Docker buildx
         run: docker buildx rm oss-buildx
         shell: bash
+
+  stop-runner:
+    name: "Stop Build EC2 Runner"
+    timeout-minutes: 10
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - docker-push # wait until all publish steps are done
+    runs-on: ubuntu-latest
+    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+    # Thus, we check for skipped here.
+    if: ${{ always() && needs.start-runner.result != 'skipped'}}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+      - name: Stop EC2 runner
+        uses: airbytehq/ec2-github-runner@base64v1.1.0
+        with:
+          mode: stop
+          github-token: ${{ env.PAT }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}