test: add required testing on Python 3.8.12 (#29116)

This shouldn't be merged until we're ready to attempt the initial full deploy, which could break beyond what we can ensure in tests.

Since this introduces a lot of additional CI I'm trying to keep the window of enforcing 3.8 tests passing in addition to 3.6 to just 3.8 tests at a minimum. The ideal situation is we merge this, deploy 3.8 to great success, then remove all of 3.6. But if the 3.8 deploy doesn't go well, we should keep enforcing 3.6 + 3.8 tests passing.

Note, visual snapshotting is disabled on 3.8, and when we make the switch to it I'll enable it in the PR and don't think there'll be any differences.

Gonna need to change repo settings again for these new check titles. And again when 3.6 is removed.

Author: joshuarli

.github/actions/setup-sentry/action.yml

@@ -54,6 +54,9 @@ outputs:
   matrix-instance-number:
     description: "The matrix instance number (starting at 1)"
     value: ${{ steps.config.outputs.matrix-instance-number }}
+  matrix-instance-total:
+    description: "Reexport of MATRIX_INSTANCE_TOTAL."
+    value: ${{ steps.config.outputs.matrix-instance-total }}
 
 runs:
   using: "composite"
@@ -63,7 +66,10 @@ runs:
       env:
         NEED_KAFKA: ${{ inputs.kafka }}
         MATRIX_INSTANCE: ${{ matrix.instance }}
-        MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
+        # XXX: We should be using something like len(strategy.matrix.instance) (not possible atm)
+        # If you have other things like python-version: [foo, bar, baz] then the sharding logic
+        # isn't right because job-total will be 3x larger and you'd never run 2/3 of the tests.
+        # MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
       run: |
         # Only set `MIGRATIONS_TEST_MIGRATE` if it is not already set (or if it's an empty string)
         if [ -z $MIGRATIONS_TEST_MIGRATE ]; then
@@ -81,11 +87,17 @@ runs:
         ### pytest-sentry configuration ###
         echo "PYTEST_SENTRY_DSN=https://6fd5cfea2d4d46b182ad214ac7810508@sentry.io/2423079" >> $GITHUB_ENV
         echo "PYTEST_ADDOPTS=--reruns 5" >> $GITHUB_ENV
+
         # this handles pytest test sharding
         if [ "$MATRIX_INSTANCE" ]; then
+          if ! [ "$MATRIX_INSTANCE_TOTAL" ]; then
+            echo "MATRIX_INSTANCE_TOTAL is required."
+            exit 1
+          fi
           echo "TEST_GROUP=$MATRIX_INSTANCE" >> $GITHUB_ENV
           echo "TOTAL_TEST_GROUPS=$MATRIX_INSTANCE_TOTAL" >> $GITHUB_ENV
         fi
+
         # This records failures on master to sentry in order to detect flakey tests, as it's
         # expected that people have failing tests on their PRs
         [ "$GITHUB_REF" = "refs/heads/master" ] && echo "PYTEST_SENTRY_ALWAYS_REPORT=1" >> $GITHUB_ENV || true
@@ -124,6 +136,7 @@ runs:
       run: |
         echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"
         echo "::set-output name=matrix-instance-number::$(($MATRIX_INSTANCE+1))"
+        echo "::set-output name=matrix-instance-total::$(($MATRIX_INSTANCE_TOTAL))"
         echo "::set-output name=acceptance-dir::.artifacts/visual-snapshots/acceptance"
 
     - name: Install python dependencies

.github/workflows/acceptance.yml

@@ -20,6 +20,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
+        # XXX: When updating this, make sure you also update CI_NODE_TOTAL.
         instance: [0, 1]
 
     env:
@@ -55,7 +56,10 @@ jobs:
         env:
           GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
           GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }}
-          CI_NODE_TOTAL: ${{ strategy.job-total }}
+          # XXX: CI_NODE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+          #      Otherwise, if there are other things in the matrix, using strategy.job-total
+          #      wouldn't be correct.
+          CI_NODE_TOTAL: 2
           CI_NODE_INDEX: ${{ matrix.instance }}
         run: |
           JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit
@@ -146,10 +150,12 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1, 2, 3]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 4
       VISUAL_SNAPSHOT_ENABLE: 1
       TEST_GROUP_STRATEGY: roundrobin
 
@@ -182,7 +188,7 @@ jobs:
         run: |
           tar xf dist.tar.gz
 
-      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: always()
         run: |
           mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
@@ -209,10 +215,13 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 1
+      TEST_GROUP_STRATEGY: roundrobin
       VISUAL_SNAPSHOT_ENABLE: 1
 
     steps:
@@ -244,7 +253,7 @@ jobs:
         run: |
           make build-chartcuterie-config
 
-      - name: Run chartcuterie tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run chartcuterie tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         run: |
           mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
           make test-chartcuterie

.github/workflows/api-docs-test.yml

@@ -14,8 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/backend-lint.yml

@@ -12,8 +12,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/backend-test.yml

@@ -13,11 +13,13 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1, 2]
 
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 3
       MIGRATIONS_TEST_MIGRATE: 1
 
     steps:
@@ -52,7 +54,7 @@ jobs:
           pip-cache-version: ${{ secrets.PIP_CACHE_VERSION }}
           snuba: true
 
-      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: steps.changes.outputs.backend == 'true'
         run: |
           # Note: `USE_SNUBA` is not used for backend tests because there are a few failing tests with Snuba enabled.

.github/workflows/backend-typing.yml

@@ -12,7 +12,7 @@ jobs:
     timeout-minutes: 12
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
+        # TODO(joshuarli): Switch to 3.8.12 when we switch over to Python 3.8.
         python-version: [3.6.13]
     steps:
       - uses: actions/checkout@v2

.github/workflows/check-if-migration-is-required.yml

@@ -25,8 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     needs: [should-check]
 
     steps:

.github/workflows/command-line-test.yml

@@ -12,8 +12,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/development-environment.yml

@@ -22,8 +22,7 @@ jobs:
         # Using Ubuntu 18 until I figure out this error:
         # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory
         os: [macos-11.0, ubuntu-18.04]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
       fail-fast: false
     env:
       PIP_DISABLE_PIP_VERSION_CHECK: on

.github/workflows/migrations.yml

@@ -41,8 +41,7 @@ jobs:
     timeout-minutes: 8
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     needs: did-migration-change
     if: ${{ needs.did-migration-change.outputs.added == 'true' }}
 

.github/workflows/plugins-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/relay-integration-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/snuba-integration-test.yml

@@ -13,10 +13,12 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 2
       USE_SNUBA: 1
       MIGRATIONS_TEST_MIGRATE: 1
       USE_INDEXER: 1
@@ -54,7 +56,7 @@ jobs:
           snuba: true
           kafka: true
 
-      - name: Run snuba test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run snuba test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: steps.changes.outputs.backend == 'true'
         run: |
           make test-snuba

.github/workflows/symbolicator-integration-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
         with:

mypy.ini

@@ -83,8 +83,8 @@ disallow_incomplete_defs=True
 check_untyped_defs=True
 disallow_untyped_decorators=True
 no_implicit_optional=True
-warn_redundant_casts=True
 warn_unused_ignores=True
+warn_redundant_casts=True
 warn_return_any=True
 no_implicit_reexport=True
 

src/sentry/utils/pytest/selenium.py

@@ -304,7 +304,8 @@ def snapshot(self, name, mobile_only=False):
         """
         # TODO(dcramer): ideally this would take the executing test package
         # into account for duplicate names
-        if os.environ.get("VISUAL_SNAPSHOT_ENABLE") != "1":
+        # TODO(josh): Change to 3.8 when it's deployed. Much easier to disable here than in GHA yml.
+        if os.environ.get("VISUAL_SNAPSHOT_ENABLE") != "1" or sys.version_info[:2] != (3, 6):
             return self
 
         self.wait_for_images_loaded()