Merge branch 'master' into ryermilov/source-amazon-ads-fix-report-str…

…eam-record-primary-keys

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.28
+current_version = 0.40.30
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.28
+VERSION=0.40.30
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -114,4 +114,4 @@ METRIC_CLIENT=
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
 USE_STREAM_CAPABLE_STATE=true
-AUTO_DETECT_SCHEMA=false
+AUTO_DETECT_SCHEMA=true

.env.dev

@@ -28,7 +28,7 @@ SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
 WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
 CRON_MICRONAUT_ENVIRONMENTS=control-plane
-AUTO_DETECT_SCHEMA=false
+AUTO_DETECT_SCHEMA=true
 
 # Sentry
 SENTRY_DSN=""

.github/CODEOWNERS

@@ -27,33 +27,44 @@ airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/Co
 airbyte-integrations/connectors/**/build.gradle # ignore gradle files for individual connectors.
 
 # Normalization
-/airbyte-integrations/bases/base-normalization/ @airbytehq/normalization
+/airbyte-integrations/bases/base-normalization/ @airbytehq/destinations
 
-# JDBC-based connectors
+# Java-based connectors
 /airbyte-integrations/bases/base-java/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-jdbc/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-alloydb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-bigquery/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-clickhouse/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-db2/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-mssql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-mysql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-oracle/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-postgres/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-redshift/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-tidb/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-jdbc/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-azure-blob-storage/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-clickhouse/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-databricks/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-gcs/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mariadb-columnstore/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mysql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-mssql/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-oracle/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-postgres/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-redshift/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/destination-rockset/ @airbytehq/jdbc-connectors
-/airbyte-integrations/connectors/source-snowflake/ @airbytehq/jdbc-connectors
+
+# Java-based source connectors
+/airbyte-integrations/bases/debezium-v1-4-2/ @airbytehq/dbsources
+/airbyte-integrations/bases/debezium-v1-9-6/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-jdbc/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-alloydb/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-bigquery/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-clickhouse/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-db2/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-mssql/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-mysql/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-oracle/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-postgres/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-redshift/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-snowflake/ @airbytehq/dbsources
+/airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources
+
+# Java-based destination connectors
+/airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations
+/airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-jdbc/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-bigquery/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-bigquery-denormalized/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-azure-blob-storage/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-clickhouse/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-databricks/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-gcs/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mariadb-columnstore/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mysql/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-mssql/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-oracle/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-postgres/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-rockset/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
+/airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations

.github/workflows/gradle.yml

@@ -124,6 +124,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Set up CI Gradle Properties
         run: |
@@ -229,6 +230,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Install Pyenv
         run: python3 -m pip install virtualenv==16.7.9 --user
@@ -412,6 +414,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Set up CI Gradle Properties
         run: |
@@ -535,6 +538,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Delete default old docker and replace it with a new one
         shell: bash
@@ -666,6 +670,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - uses: actions/setup-node@v3
         with:
@@ -800,6 +805,7 @@ jobs:
         if: always()
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Publish Platform Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -945,6 +951,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Install unzip
         shell: bash
@@ -1021,6 +1028,7 @@ jobs:
         if: always()
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - name: Publish Kube Test Results
         id: kube-results
@@ -1191,6 +1199,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+          token: ${{ env.PAT }}
 
       - uses: actions/setup-java@v1
         with:

.github/workflows/publish-cdk-command-manually.yml

@@ -20,7 +20,7 @@ on:
           - patch
         required: true
       skip-publish-test:
-        description: 'By default, the job publishes to Test PyPi. Use "true" to only publish to actual PyPi servers.'
+        description: 'By default, the job publishes to both Test and actual PyPi. Use "true" to only publish to actual PyPi servers.'
         required: false
       changelog-message:
         description: "Changelog message to be added to CHANGELOG.md"
@@ -178,14 +178,14 @@ jobs:
                         "type": "section",
                         "text": {
                             "type": "mrkdwn",
-                            "text": "A new version of Airbyte CDK has been released!\n\n"
+                            "text": "A new version of Airbyte CDK has been released with changelog: ${{ github.event.inputs.changelog-message }}!\n\n"
                         }
                     },
                     {
                         "type": "section",
                         "text": {
                             "type": "mrkdwn",
-                            "text": "See details on <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md?plain=1#L3-L4|CHANGELOG.md> and <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
+                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
                         }
                     }
                 ]

.github/workflows/run-qa-engine.yml

@@ -0,0 +1,32 @@
+name: Run QA Engine
+
+on:
+  workflow_dispatch:
+  schedule:
+    # 1pm UTC is 6am PDT.
+    # same time as Generate Build Report
+    - cron: "0 13 * * *"
+
+jobs:
+  run-qa-engine:
+    name: "Run QA Engine"
+    if: github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Setup Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.PROD_SPEC_CACHE_SA_KEY }}
+          export_default_credentials: true
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install ci-connector-ops package
+        run: pip install --quiet -e ./tools/ci_connector_ops
+      - name: Run QA Engine
+        env:
+          QA_ENGINE_AIRBYTE_DATA_PROD_SA: '${{ secrets.QA_ENGINE_AIRBYTE_DATA_PROD_SA }}'
+        run: run-qa-engine

.github/workflows/test-command.yml

@@ -95,19 +95,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
-      - name: Install Pyenv
-        run: |
-          python3 -m pip install --quiet virtualenv==16.7.9 --user
-          python3 -m virtualenv venv
-          source venv/bin/activate
       - name: Install CI scripts
         # all CI python packages have the prefix "ci_"
         run: |
-          source venv/bin/activate
           pip install --quiet -e ./tools/ci_*
       - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
         run: |
-          source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }} write-to-storage
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
@@ -117,7 +110,6 @@ jobs:
           fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
-
       - name: Test ${{ github.event.inputs.connector }}
         id: test
         env:
@@ -132,7 +124,6 @@ jobs:
       - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
         if: always()
         run: |
-          source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }} update-secrets
           # normalization also runs destination-specific tests, so fetch their creds also
           if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
@@ -155,7 +146,6 @@ jobs:
             **/normalization_test_output/**/build/compiled/airbyte_utils/**
             **/normalization_test_output/**/build/run/airbyte_utils/**
             **/normalization_test_output/**/models/generated/**
-
       - name: Test coverage reports artifacts
         if: github.event.inputs.comment-id && success()
         uses: actions/upload-artifact@v3
@@ -164,10 +154,14 @@ jobs:
           path: |
             **/${{ github.event.inputs.connector }}/htmlcov/**
           retention-days: 3
-
+      - name: Run QA checks for ${{ github.event.inputs.connector }}
+        id: qa_checks
+        if: always()
+        run: |
+          run-qa-checks ${{ github.event.inputs.connector }}
       - name: Report Status
         if: github.ref == 'refs/heads/master' && always()
-        run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}}
+        run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}} ${{steps.qa_checks.outcome}}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }}

.vscode/settings.json

@@ -41,5 +41,6 @@
     "editor.codeActionsOnSave": {
       "source.fixAll.stylelint": true
     }
-  }
+  },
+  "typescript.tsdk": "./airbyte-webapp/node_modules/typescript/lib"
 }