Merge branch 'master' into artem1205/source-amazon-seller-partner-165…

…8-max-time
airbytehq · Nov 14, 2023 · 83a6ea1 · 83a6ea1
2 parents 6b70d9d + f6bc4b8
commit 83a6ea1
Show file tree

Hide file tree

Showing 177 changed files with 2,689 additions and 1,304 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -38,6 +38,9 @@
 /airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources
 
 # Java-based destination connectors
+airbyte-cdk/java/airbyte-cdk/db-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/s3-destinations/ @airbytehq/destinations
+airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations
 /airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
 /airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
 /airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations

diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml
@@ -104,7 +104,7 @@ runs:
       shell: bash
       run: |
         export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock"
-        airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+        airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
         _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
         CI_CONTEXT: "${{ inputs.context }}"

diff --git a/.github/workflows/airbyte-ci-release-experiment.yml b/.github/workflows/airbyte-ci-release-experiment.yml
diff --git a/.github/workflows/airbyte-ci-release.yml b/.github/workflows/airbyte-ci-release.yml
@@ -0,0 +1,130 @@
+name: Connector Ops CI - Airbyte CI Release
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - "airbyte-ci/connectors/pipelines/**"
+  workflow_dispatch:
+
+env:
+  DEV_GCS_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service
+  PROD_GCS_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service
+  BINARY_FILE_NAME: airbyte-ci
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu", "macos"]
+
+    steps:
+      - name: Checkout Airbyte
+        id: checkout_airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.sha }} # This is required to make sure that the same commit is checked out on all runners
+
+      - name: Get short SHA
+        id: get_short_sha
+        uses: benjlevesque/short-sha@v2.2
+
+      - name: Install Python
+        id: install_python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Poetry
+        id: install_poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install Dependencies
+        id: install_dependencies
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry install --with dev
+
+      - name: Build release
+        id: build_release
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }}
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+
+      - name: Authenticate to Google Cloud Dev
+        id: auth_dev
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}"
+
+      - name: Upload pre-release to GCS dev bucket
+        id: upload_pre_release_to_gcs
+        if: github.ref != 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print pre-release public url
+        id: print_pre_release_public_url
+        run: |
+          echo "https://storage.googleapis.com/${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}/${{ env.BINARY_FILE_NAME }}"
+
+      # if master, upload per version and latest to prod bucket
+
+      - name: Set version from poetry version --short
+        id: set_version
+        if: github.ref == 'refs/heads/master'
+        working-directory: airbyte-ci/connectors/pipelines/
+        run: |
+          echo "::set-output name=version::$(poetry version --short)"
+
+      - name: Authenticate to Google Cloud Prod
+        id: auth_prod
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: "${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}"
+
+      - name: Upload version release to GCS prod bucket
+        id: upload_version_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print release version public url
+        id: print_version_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}/${{ env.BINARY_FILE_NAME }}"
+
+      - name: Upload latest release to GCS prod bucket
+        id: upload_latest_release_to_gcs
+        if: github.ref == 'refs/heads/master'
+        uses: google-github-actions/upload-cloud-storage@v1
+        with:
+          path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }}
+          destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest
+          headers: |-
+            cache-control:public, max-age=10
+
+      - name: Print latest release public url
+        id: print_latest_release_public_url
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest/${{ env.BINARY_FILE_NAME }}"
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -1,7 +1,9 @@
-name: Format Code (Python + Java)
+name: Check for formatting errors
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
+  # Cancel any previous runs on the same branch if they are still in progress
+  cancel-in-progress: true
 
 on:
   workflow_dispatch:
@@ -10,8 +12,8 @@ on:
       - master
   pull_request:
 jobs:
-  format-and-commit:
-    runs-on: ubuntu-latest
+  format-fix:
+    runs-on: "conn-prod-xlarge-runner"
     name: "Apply All Formatting Rules"
     timeout-minutes: 40
     steps:
@@ -20,16 +22,16 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
           # Important that this is set so that CI checks are triggered again
-          # Without this we would be be forever waiting on required checks to pass
+          # Without this we would be forever waiting on required checks to pass
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
 
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
-      # even if the workflow is skipped. If we used github actions filters, the workflow
+      # IMPORTANT! This is necessary to make sure that a status is reported on the PR
+      # even if the workflow is skipped. If we used GitHub Actions filters, the workflow
       # would not be reported as skipped, but instead would be forever pending.
       #
       # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
       #
-      # Also it gets worse
+      # Also, it gets worse
       #
       # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
       # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
@@ -42,59 +44,35 @@ jobs:
               - '**/*'
               - '!**/*.md'
 
-      - uses: actions/setup-java@v3
-        with:
-          distribution: "zulu"
-          java-version: "17"
-
-      - uses: actions/setup-python@v4
+      - name: Run airbyte-ci format fix
+        uses: ./.github/actions/run-dagger-pipeline
         with:
-          python-version: "3.10"
-
-      - name: Set up CI Gradle Properties
-        run: |
-          mkdir -p ~/.gradle/
-          cat > ~/.gradle/gradle.properties <<EOF
-          org.gradle.jvmargs=-Xmx8g -Xss4m \
-            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-          org.gradle.workers.max=8
-          org.gradle.vfs.watch=false
-          EOF
-
-      - name: Format
-        if: steps.changes.outputs.format_any_changed == 'true'
-        uses: Wandalen/wretry.action@v1.0.42
-        with:
-          command: ./gradlew format --scan --info --stacktrace
-          attempt_limit: 3
-          attempt_delay: 5000 # in ms
+          context: "pull_request"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          subcommand: "format fix all"
 
       # This is helpful in the case that we change a previously committed generated file to be ignored by git.
       - name: Remove any files that have been gitignored
         run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
 
       - name: Commit Formatting Changes (PR)
-        uses: stefanzweifel/git-auto-commit-action@v4
+        uses: stefanzweifel/git-auto-commit-action@v5
         # do not commit if master branch
         if: github.ref != 'refs/heads/master'
         with:
           commit_message: Automated Commit - Formatting Changes
           commit_user_name: Octavia Squidington III
           commit_user_email: octavia-squidington-iii@users.noreply.github.com
 
-      - name: "Fail on Formatting Changes (Master)"
-        if: github.ref == 'refs/heads/master'
-        run: git --no-pager diff && test -z "$(git --no-pager diff)"
-
   notify-failure-slack-channel:
     name: "Notify Slack Channel on Build Failures"
     runs-on: ubuntu-latest
     needs:
-      - format-and-commit
+      - format-fix
     if: ${{ failure() && github.ref == 'refs/heads/master' }}
     steps:
       - name: Checkout Airbyte

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -103,8 +103,7 @@ jobs:
         with:
           read-only: ${{ github.ref != 'refs/heads/master' }}
           # TODO: be able to remove the skipSlowTests property
-          # TODO: remove the format task ASAP
-          arguments: --scan --no-daemon --no-watch-fs format check -DskipSlowTests=true
+          arguments: --scan --no-daemon --no-watch-fs check -DskipSlowTests=true
 
   # In case of self-hosted EC2 errors, remove this block.
   stop-check-runner: