diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml
index a15861ee47..b7e14f72e8 100644
--- a/.github/workflows/upstream.yml
+++ b/.github/workflows/upstream.yml
@@ -10,103 +10,101 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PIP_ONLY_BINARY: numpy
+  PIP_ONLY_BINARY: ":all:"
   # For cmake:
   VERBOSE: 1
 
 jobs:
   standard:
-    name: "🐍 3.11 latest internals • ubuntu-latest • x64"
+    name: "🐍 3.12 latest • ubuntu-latest • x64"
     runs-on: ubuntu-latest
+    # Only runs when the  'python dev' label is selected
     if: "contains(github.event.pull_request.labels.*.name, 'python dev')"
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python 3.11
+    - name: Setup Python 3.12
       uses: actions/setup-python@v4
       with:
-        python-version: "3.11-dev"
+        python-version: "3.12-dev"
 
-    - name: Setup Boost (Linux)
-      if: runner.os == 'Linux'
+    - name: Setup Boost
       run: sudo apt-get install libboost-dev
 
     - name: Update CMake
       uses: jwlawson/actions-setup-cmake@v1.13
 
-    - name: Prepare env
+    - name: Run pip installs
       run: |
+        python -m pip install --upgrade pip
         python -m pip install -r tests/requirements.txt
 
-    - name: Setup annotations on Linux
-      if: runner.os == 'Linux'
-      run: python -m pip install pytest-github-actions-annotate-failures
+    - name: Show platform info
+      run: |
+        python -m platform
+        cmake --version
+        pip list
 
     # First build - C++11 mode and inplace
     - name: Configure C++11
       run: >
-        cmake -S . -B .
+        cmake -S . -B build11
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=11
+        -DCMAKE_BUILD_TYPE=Debug
 
     - name: Build C++11
-      run: cmake --build . -j 2
+      run: cmake --build build11 -j 2
 
     - name: Python tests C++11
-      run: cmake --build . --target pytest -j 2
+      run: cmake --build build11 --target pytest -j 2
 
-    - name: C++11 tests
-      run: cmake --build .  --target cpptest -j 2
+    # - name: C++11 tests
+    #   run: cmake --build build11  --target cpptest -j 2
 
     - name: Interface test C++11
-      run: cmake --build . --target test_cmake_build
-
-    - name: Clean directory
-      run: git clean -fdx
+      run: cmake --build build11 --target test_cmake_build
 
     # Second build - C++17 mode and in a build directory
     - name: Configure C++17
       run: >
-        cmake -S . -B build2
+        cmake -S . -B build17
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
-        ${{ matrix.args }}
-        ${{ matrix.args2 }}
 
     - name: Build
-      run: cmake --build build2 -j 2
+      run: cmake --build build17 -j 2
 
     - name: Python tests
-      run: cmake --build build2 --target pytest
+      run: cmake --build build17 --target pytest
 
-    - name: C++ tests
-      run: cmake --build build2 --target cpptest
+    # - name: C++ tests
+    #   run: cmake --build build17 --target cpptest
 
     # Third build - C++17 mode with unstable ABI
     - name: Configure (unstable ABI)
       run: >
-        cmake -S . -B build3
+        cmake -S . -B build17max
         -DPYBIND11_WERROR=ON
         -DDOWNLOAD_CATCH=ON
         -DDOWNLOAD_EIGEN=ON
         -DCMAKE_CXX_STANDARD=17
         -DPYBIND11_INTERNALS_VERSION=10000000
         "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp"
-        ${{ matrix.args }}
 
     - name: Build (unstable ABI)
-      run: cmake --build build3 -j 2
+      run: cmake --build build17max -j 2
 
     - name: Python tests (unstable ABI)
-      run: cmake --build build3 --target pytest
+      run: cmake --build build17max --target pytest
 
     - name: Interface test
-      run: cmake --build build3 --target test_cmake_build
+      run: cmake --build build17max --target test_cmake_build
 
     # This makes sure the setup_helpers module can build packages using
     # setuptools
diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py
index 7f80a5da56..1b4c89add1 100644
--- a/tests/test_exceptions.py
+++ b/tests/test_exceptions.py
@@ -317,6 +317,11 @@ def test_error_already_set_what_with_happy_exceptions(
     assert what == expected_what
 
 
+@pytest.mark.skipif(
+    # Intentionally very specific:
+    "sys.version_info == (3, 12, 0, 'alpha', 6)",
+    reason="WIP: https://github.com/python/cpython/issues/102594",
+)
 @pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault")
 def test_flaky_exception_failure_point_init():
     with pytest.raises(RuntimeError) as excinfo: