Upgrade to Python 3.9 (#11763)

* Dockerfile to 3.9

* Python version

* More python updates

* 3.9 on GitHub actions and lint updates

* Test out 3.9.11 on GitHub actions

* install python with an action

* formatting: newline

* Also has python code

* only check first level for changed modules
Previous example (source-google-search-console/credentials)

* Test failure: there is no logger.trace

.github/actions/ci-tests-runner/action.yml

@@ -26,16 +26,15 @@ inputs:
 runs:
   using: "composite"
   steps:
-
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.9
 
     - name: Install Java
       uses: actions/setup-java@v1
       with:
-        java-version: '17'
+        java-version: "17"
 
     - name: Tests of CI
       shell: bash
@@ -57,20 +56,20 @@ runs:
       id: gcloud-tunnel
       shell: bash
       run: |
-          while true; do
-            PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
-            status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
-            if [ "${status}" != "0" ]; then
-                echo "$PORT is free to use";
-                break;
-            fi
-          done
-          IPS=($(hostname -I))
-          LOCAL_IP_PORT="${IPS[0]}:${PORT}"
-          gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing &
-          echo ::set-output name=pid::$!
-          echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/"
-          echo "::echo::on"
+        while true; do
+          PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
+          status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
+          if [ "${status}" != "0" ]; then
+              echo "$PORT is free to use";
+              break;
+          fi
+        done
+        IPS=($(hostname -I))
+        LOCAL_IP_PORT="${IPS[0]}:${PORT}"
+        gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing &
+        echo ::set-output name=pid::$!
+        echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/"
+        echo "::echo::on"
 
     - name: Python Tests
       id: ci-py-tests
@@ -88,10 +87,6 @@ runs:
         module-name: ${{ inputs.module-name }}
         module-folder: ${{ inputs.module-folder }}
 
-
-
-
-
     - name: Prepare SQ Options
       shell: bash
       id: sq-options
@@ -142,25 +137,24 @@ runs:
         MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
         echo "::set-output name=module_dir::${MODULE_DIR}"
 
-
     - name: SonarQube Scan
 
       uses: sonarsource/sonarqube-scan-action@master
       env:
         SONAR_TOKEN: ${{ inputs.sonar-token }}
         SONAR_HOST_URL: ${{ steps.gcloud-tunnel.outputs.sonar-host }}
       with:
-          projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }}
-          args: >
-            -Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }}
-            -Dsonar.verbose=true
-            -Dsonar.working.directory=/tmp/scannerwork
-            -Dsonar.language=${{ inputs.module-lang }}
-            -Dsonar.sourceEncoding=UTF-8
-            -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
-            -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py
-            -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
-            ${{ steps.sq-options.outputs.options }}
+        projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }}
+        args: >
+          -Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }}
+          -Dsonar.verbose=true
+          -Dsonar.working.directory=/tmp/scannerwork
+          -Dsonar.language=${{ inputs.module-lang }}
+          -Dsonar.sourceEncoding=UTF-8
+          -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
+          -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py
+          -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
+          ${{ steps.sq-options.outputs.options }}
 
     - name: Generate SonarQube Report
       shell: bash

.github/workflows/gradle.yml

@@ -80,8 +80,6 @@ jobs:
             ${{ secrets.SUPERTOPHER_PAT }} \
             ${{ secrets.DAVINCHIA_PAT }}
 
-
-
   # Uncomment to debug.
   #  changes-output:
   #    name: "Debug Change Detection Logic"
@@ -114,11 +112,9 @@ jobs:
         with:
           java-version: "17"
 
-      # octavia-cli install and testing requires Python.
-      # We use 3.8 in this project because 3.7 is not supported on Apple M1.
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Set up CI Gradle Properties
         run: |
@@ -221,7 +217,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.9"
 
       - name: Install Pyenv
         run: python3 -m pip install virtualenv==16.7.9 --user

.github/workflows/publish-cdk-command.yml

@@ -14,11 +14,10 @@ on:
         description: 'By default dry-run publishes to Test PyPi. Use "false" to publish to actual PyPi servers.'
         required: false
       comment-id:
-        description: 'The comment-id of the slash command. Used to update the comment with the status.'
+        description: "The comment-id of the slash command. Used to update the comment with the status."
         required: false
 
 jobs:
-
   build-cdk:
     runs-on: ubuntu-latest
     strategy:
@@ -31,7 +30,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - uses: actions/setup-java@v1
         with:
-          java-version: '17'
+          java-version: "17"
       - name: Checkout Airbyte
         uses: actions/checkout@v2
         with:
@@ -74,9 +73,9 @@ jobs:
         uses: mariamrf/py-package-publish-action@v1.1.0
         with:
           # specify the same version as in ~/.python-version
-          python_version: '3.7.9'
-          pip_version: '21.1'
-          subdir: 'airbyte-cdk/python/'
+          python_version: "3.9.11"
+          pip_version: "21.1"
+          subdir: "airbyte-cdk/python/"
         env:
           TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
           TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}

.github/workflows/publish-command.yml

@@ -104,8 +104,11 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: "17"
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
       - name: Install Pyenv and Tox
-        # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/
         run: |
           python3 -m pip install --quiet virtualenv==16.7.9 --user
           python3 -m virtualenv venv

.github/workflows/release-airbyte-os.yml

@@ -66,7 +66,7 @@ jobs:
       # necessary to install pip
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - name: Release Airbyte
         id: release_airbyte
         env:
@@ -79,7 +79,6 @@ jobs:
 
   # We are releasing octavia from a separate job because:
   # - The self hosted runner used in releaseAirbyte does not have the docker buildx command to build multi-arch images
-  # - Plaform build requires Python 3.7, Octavia Build requires 3.8
   releaseOctavia:
     runs-on: ubuntu-latest
     environment: more-secrets
@@ -95,11 +94,9 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: "16.13.0"
-      # octavia-cli build requires Python 3.8.
-      # We use 3.8 in this project because 3.7 is not supported on Apple M1.
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Release Octavia
         id: release_octavia
         env:
@@ -121,7 +118,7 @@ jobs:
       # necessary to install pip
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - name: Bump version
         id: bump_version
         env:

.github/workflows/sonar-scan.yml

@@ -4,7 +4,6 @@ on:
     types: [opened, synchronize, closed]
 
 jobs:
-
   detect-changes:
     name: Detect Changed Modules
     timeout-minutes: 5
@@ -23,11 +22,11 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
-      - name: Intall Requirements
+          python-version: 3.9
+      - name: Install Requirements
         run: |
-           pip install ./tools/ci_common_utils ./tools/ci_code_validator[tests]
-           pytest ./tools/ci_code_validator
+          pip install ./tools/ci_common_utils ./tools/ci_code_validator[tests]
+          pytest ./tools/ci_code_validator
       - name: Detect Changed Modules
         id: detect-changed-modules
         run: |
@@ -39,7 +38,7 @@ jobs:
     # Do only run if the PR is not a draft and the changed modules matrix contains at least one entry
     if: github.event.pull_request.draft == false && fromJson(needs.detect-changes.outputs.changed-modules).include[0] != null
     needs: detect-changes
-    name:  Tests for ${{ matrix.module }}
+    name: Tests for ${{ matrix.module }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -50,29 +49,25 @@ jobs:
       MODULE_FOLDER: ${{ matrix.folder }}
       ENV_NAME: "github"
 
-
     steps:
-    - name: Print Settings
-      run: |
-         echo "Module: ${{ env.MODULE_NAME }}, Lang: ${{ env.MODULE_LANG }}, Folder: ${{ env.MODULE_FOLDER }}"
-    - name: Checkout Airbyte
-      if: ${{ env.ENV_NAME == 'github' }}
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Run Tests Runner
-      id: run-python-tests
-      uses: ./.github/actions/ci-tests-runner
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        module-name: ${{ env.MODULE_NAME }}
-        module-folder: ${{ env.MODULE_FOLDER }}
-        module-lang: ${{ env.MODULE_LANG }}
-        sonar-token: ${{ secrets.SONAR_TOKEN }}
-        sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }}
-        pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}"
-        remove-sonar-project: true
-
-
+      - name: Print Settings
+        run: |
+          echo "Module: ${{ env.MODULE_NAME }}, Lang: ${{ env.MODULE_LANG }}, Folder: ${{ env.MODULE_FOLDER }}"
+      - name: Checkout Airbyte
+        if: ${{ env.ENV_NAME == 'github' }}
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
+      - name: Run Tests Runner
+        id: run-python-tests
+        uses: ./.github/actions/ci-tests-runner
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          module-name: ${{ env.MODULE_NAME }}
+          module-folder: ${{ env.MODULE_FOLDER }}
+          module-lang: ${{ env.MODULE_LANG }}
+          sonar-token: ${{ secrets.SONAR_TOKEN }}
+          sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }}
+          pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}"
+          remove-sonar-project: true

.github/workflows/test-command.yml

@@ -90,8 +90,11 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: "17"
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
       - name: Install Pyenv and Tox
-        # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/
         run: |
           python3 -m pip install --quiet virtualenv==16.7.9 --user
           python3 -m virtualenv venv

.github/workflows/test-performance-command.yml

@@ -95,9 +95,12 @@ jobs:
       - name: Install Java
         uses: actions/setup-java@v1
         with:
-          java-version: '17'
+          java-version: "17"
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
       - name: Install Pyenv and Tox
-        # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/
         run: |
           python3 -m pip install --quiet virtualenv==16.7.9 --user
           python3 -m virtualenv venv

.python-version

@@ -1 +1 @@
-3.7.9
+3.9.11

.readthedocs.yaml

@@ -12,7 +12,7 @@ sphinx:
 
 # Set the version of Python and requirements required to build your docs
 python:
-  version: "3.7"
+  version: "3.9"
   install:
     - method: pip
       path: airbyte-cdk/python

airbyte-cdk/python/README.md

@@ -49,7 +49,7 @@ See the [concepts docs](docs/concepts/) for a tour through what the API offers.
 
 ### First time setup
 
-We assume `python` points to python >=3.7.
+We assume `python` points to python >=3.9.
 
 Setup a virtual env:
 

airbyte-cdk/python/docs/tutorials/cdk-tutorial-any-percent/cdk-speedrun.md

@@ -4,7 +4,7 @@ This is a blazing fast guide to building an HTTP source connector. Think of it a
 
 # Dependencies
 
-1. Python >= 3.7
+1. Python >= 3.9
 2. Docker
 3. NodeJS
 

airbyte-cdk/python/docs/tutorials/cdk-tutorial-python-http/0-getting-started.md

@@ -6,11 +6,11 @@ This is a step-by-step guide for how to create an Airbyte source in Python to re
 
 ## Requirements
 
-* Python >= 3.7
+* Python >= 3.9
 * Docker
 * NodeJS \(only used to generate the connector\). We'll remove the NodeJS dependency soon.
 
-All the commands below assume that `python` points to a version of python >=3.7.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`.
+All the commands below assume that `python` points to a version of python >=3.9.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`.
 
 ## Checklist
 

airbyte-cdk/python/docs/tutorials/http_api_source.md

@@ -7,11 +7,11 @@ Exchangerates API as an example since it is both simple but demonstrates a lot o
 
 ## Requirements
 
-* Python >= 3.7 
+* Python >= 3.9 
 * Docker
 * NodeJS (only used to generate the connector). We'll remove the NodeJS dependency soon.
 
-All the commands below assume that `python` points to a version of python >=3.7.0. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`.
+All the commands below assume that `python` points to a version of python >=3.9. On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3`.
 
 ## Checklist
 * Step 1: Create the source using the template

airbyte-cdk/python/setup.py

@@ -57,7 +57,7 @@
         "vcrpy",
         "Deprecated~=1.2",
     ],
-    python_requires=">=3.7.0",
+    python_requires=">=3.9",
     extras_require={
         "dev": [
             "MyPy~=0.812",