Merge branch 'master' into tcho/speakeasy-route-cloud

.env

@@ -113,3 +113,5 @@ METRIC_CLIENT=
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
 USE_STREAM_CAPABLE_STATE=true
+AUTO_DETECT_SCHEMA=false
+

.env.dev

@@ -28,6 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
 CRON_MICRONAUT_ENVIRONMENTS=control-plane
+AUTO_DETECT_SCHEMA=false
 
 # Sentry
 SENTRY_DSN=""

.github/CODEOWNERS

@@ -9,14 +9,28 @@
 /airbyte-config/init/src/main/resources/icons/*.svg @airbytehq/design
 
 # CDK and SAT
-/airbyte-cdk/ @airbytehq/api-connectors-dx
-/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/api-connectors-dx
-/airbyte-integrations/connector-templates/ @airbytehq/api-connectors-dx
+/airbyte-cdk/ @airbytehq/connector-extensibility
+/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/connector-extensibility
+/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
 
 # Protocol related items
 /airbyte-protocol/ @airbytehq/protocol-reviewers
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Airbyte Maintainers
+airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/BaseConfigDatabaseTest.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/ @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @airbytehq/airbyte-maintainers
+airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/ @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryReader.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SecretsRepositoryWriter.java @airbytehq/airbyte-maintainers
+airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @airbytehq/airbyte-maintainers
+*build.gradle @airbytehq/airbyte-maintainers
+airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors.
+
 # Normalization
 /airbyte-integrations/bases/base-normalization/ @airbytehq/normalization
 
@@ -47,4 +61,4 @@
 /airbyte-integrations/connectors/destination-postgres/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/destination-rockset/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
+/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors

.github/actions/start-aws-runner/action.yml

@@ -8,8 +8,8 @@ inputs:
   github-token:
     required: true
   ec2-image-id:
-    # github-self-hosted-runner-ubuntu-20-100g-disk-with-cypress-deps
-    default: "ami-005924fb76f7477ce"
+    # https://internal-docs.airbyte.io/Things-To-Know/Build-Runner-Images#ami-05b3422b3649a6911
+    default: "ami-05b3422b3649a6911"
     required: true
   ec2-instance-type:
     default: "c5.2xlarge"

.github/workflows/gradle.yml

@@ -134,20 +134,35 @@ jobs:
           EOF
 
       - name: Format
-        run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: ./tools/bin/check_for_file_changes
 
       - name: Build
-        run: |
-          SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run integration tests
-        run: ./tools/bin/integration_tests_octavia.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/integration_tests_octavia.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
   # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
@@ -230,13 +245,25 @@ jobs:
           EOF
 
       - name: Generate Template scaffold
-        run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Format
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Build
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
@@ -392,7 +419,11 @@ jobs:
           EOF
 
       - name: Build :airbyte-webapp
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Publish Storybook to Chromatic
         uses: chromaui/action@v1
@@ -447,12 +478,20 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run End-to-End Frontend Tests
         env:
           CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }}
-        run: ./tools/bin/e2e_test.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/e2e_test.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
   # In case of self-hosted EC2 errors, remove this block.
   stop-frontend-runner:
     name: "Frontend: Stop Runner"
@@ -565,25 +604,43 @@ jobs:
           EOF
 
       - name: Format
-        run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
       - name: Build
-        run: |
-          SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Integration test
-        run: |
-          SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Slow integration test
         if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
-        run: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Test if Seed spec is updated
-        run: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       # todo (cgardens) - scope by platform.
       - name: Check documentation
@@ -595,10 +652,18 @@ jobs:
       #        run: docker run --rm busybox df -h
 
       - name: Run End-to-End Acceptance Tests
-        run: ./tools/bin/acceptance_test.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/acceptance_test.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Automatic Migration Acceptance Test
-        run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - uses: actions/setup-python@v4
         if: always()
@@ -792,7 +857,11 @@ jobs:
           CHANGE_MINIKUBE_NONE_USER: true
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:
@@ -802,8 +871,11 @@ jobs:
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - uses: actions/setup-python@v4
         if: always()
@@ -961,14 +1033,17 @@ jobs:
           fetch-depth: 0
 
       - name: Delete default old docker and replace it with a new one
-        shell: bash
-        run: |
-          sudo rm /var/lib/dpkg/lock
-          sudo rm /var/lib/dpkg/lock-frontend
-          sudo apt-get remove docker.io || sudo apt-get remove docker
-          curl -fsSL https://get.docker.com | bash -
-          sudo rm -f /var/lib/dpkg/lock
-          sudo rm -f /var/lib/dpkg/lock-frontend
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            sudo rm /var/lib/dpkg/lock
+            sudo rm /var/lib/dpkg/lock-frontend
+            sudo apt-get remove docker.io || sudo apt-get remove docker
+            curl -fsSL https://get.docker.com | bash -
+            sudo rm -f /var/lib/dpkg/lock
+            sudo rm -f /var/lib/dpkg/lock-frontend
+          attempt_limit: 3
+          attempt_delay: 2000 # in ms
 
       - name: Cache Build Artifacts
         uses: ./.github/actions/cache-build-artifacts
@@ -1004,13 +1079,16 @@ jobs:
 
       - name: Install tmate and it's dependencies
         if: inputs.debug_mode == 'true'
-        shell: bash
-        run: |
-          sudo rm /var/lib/dpkg/lock
-          sudo rm /var/lib/dpkg/lock-frontend
-          sudo apt-get -y install tmate
-          sudo rm -f /var/lib/dpkg/lock
-          sudo rm -f /var/lib/dpkg/lock-frontend
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            sudo rm /var/lib/dpkg/lock
+            sudo rm /var/lib/dpkg/lock-frontend
+            sudo apt-get -y install tmate
+            sudo rm -f /var/lib/dpkg/lock
+            sudo rm -f /var/lib/dpkg/lock-frontend
+          attempt_limit: 3
+          attempt_delay: 2000 # in ms
 
       - name: Start tmate session in background
         if: inputs.debug_mode == 'true'
@@ -1031,7 +1109,11 @@ jobs:
           extra-setup-args: --docker --kube-apiserver-arg service-node-port-range=6000-32767
 
       - name: Try the cluster !
-        run: kubectl get pods -A
+        uses: Wandalen/wretry.action@master
+        with:
+          command: kubectl get pods -A
+          attempt_limit: 3
+          attempt_delay: 3000 # in ms
 
       - name: Set up CI Gradle Properties
         run: |
@@ -1047,7 +1129,11 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run Helm End-to-End Acceptance Tests
         env:
@@ -1058,8 +1144,11 @@ jobs:
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         timeout-minutes: 20
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Generate Test Report
         uses: dorny/test-reporter@v1
@@ -1074,6 +1163,16 @@ jobs:
         with:
           name: Kubernetes Logs
           path: /tmp/kubernetes_logs/*
+
+      - name: Upload test results to BuildPulse for flaky test detection
+        if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        uses: Workshop64/buildpulse-action@main
+        with:
+          account: 59758427
+          repository: 283046497
+          path: "/actions-runner/_work/airbyte/airbyte/*"
+          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
 
 
   # # In case of self-hosted EC2 errors, remove this block.