Merge branch 'main' into deprecate_writer

vstinner · Feb 19, 2025 · a73e7fc · a73e7fc
2 parents d650aae + 2498c22
commit a73e7fc
Show file tree

Hide file tree

Showing 536 changed files with 23,702 additions and 11,352 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -106,6 +106,7 @@ Objects/exceptions.c          @iritkatriel
 
 # Hashing & cryptographic primitives
 **/*hashlib*                  @gpshead @tiran @picnixz
+**/*hashopenssl*              @gpshead @tiran @picnixz
 **/*pyhash*                   @gpshead @tiran
 **/sha*                       @gpshead @tiran @picnixz
 Modules/md5*                  @gpshead @tiran @picnixz

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -22,25 +22,25 @@ env:
   FORCE_COLOR: 1
 
 jobs:
-  check_source:
+  build-context:
     name: Change detection
     # To use boolean outputs from this job, parse them as JSON.
     # Here's some examples:
     #
-    #   if: fromJSON(needs.check_source.outputs.run-docs)
+    #   if: fromJSON(needs.build-context.outputs.run-docs)
     #
     #   ${{
-    #        fromJSON(needs.check_source.outputs.run_tests)
+    #        fromJSON(needs.build-context.outputs.run-tests)
     #        && 'truthy-branch'
     #        || 'falsy-branch'
     #   }}
     #
-    uses: ./.github/workflows/reusable-change-detection.yml
+    uses: ./.github/workflows/reusable-context.yml
 
   check-docs:
     name: Docs
-    needs: check_source
-    if: fromJSON(needs.check_source.outputs.run-docs)
+    needs: build-context
+    if: fromJSON(needs.build-context.outputs.run-docs)
     uses: ./.github/workflows/reusable-docs.yml
 
   check_autoconf_regen:
@@ -51,8 +51,8 @@ jobs:
     container:
       image: ghcr.io/python/autoconf:2025.01.02.12581854023
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     steps:
       - name: Install Git
         run: |
@@ -62,8 +62,6 @@ jobs:
         with:
           fetch-depth: 1
           persist-credentials: false
-      - name: Runner image version
-        run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
       - name: Check Autoconf and aclocal versions
         run: |
           grep "Generated by GNU Autoconf 2.72" configure
@@ -94,8 +92,8 @@ jobs:
     # reproducible: to get the same tools versions (autoconf, aclocal, ...)
     runs-on: ubuntu-24.04
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -104,14 +102,14 @@ jobs:
         with:
           python-version: '3.x'
       - name: Runner image version
-        run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+        run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
       - name: Restore config.cache
         uses: actions/cache@v4
         with:
           path: config.cache
           # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
-          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
-      - name: Install Dependencies
+          key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }}
+      - name: Install dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
         run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
@@ -153,8 +151,8 @@ jobs:
     name: >-
       Windows
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
-    needs: check_source
-    if: fromJSON(needs.check_source.outputs.run_tests)
+    needs: build-context
+    if: fromJSON(needs.build-context.outputs.run-windows-tests)
     strategy:
       fail-fast: false
       matrix:
@@ -184,8 +182,8 @@ jobs:
   build_windows_msi:
     name: >-  # ${{ '' } is a hack to nest jobs under the same sidebar category
       Windows MSI${{ '' }}
-    needs: check_source
-    if: fromJSON(needs.check_source.outputs.run-win-msi)
+    needs: build-context
+    if: fromJSON(needs.build-context.outputs.run-windows-msi)
     strategy:
       matrix:
         arch:
@@ -200,8 +198,8 @@ jobs:
     name: >-
       macOS
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -226,7 +224,7 @@ jobs:
           free-threading: true
     uses: ./.github/workflows/reusable-macos.yml
     with:
-      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      config_hash: ${{ needs.build-context.outputs.config-hash }}
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
 
@@ -235,8 +233,8 @@ jobs:
       Ubuntu
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
       ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }}
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       matrix:
         bolt:
@@ -247,17 +245,17 @@ jobs:
         - true
         os:
         - ubuntu-24.04
-        - ubuntu-24.04-arm
+        - ubuntu-22.04-arm
         exclude:
         # Do not test BOLT with free-threading, to conserve resources
         - bolt: true
           free-threading: true
         # BOLT currently crashes during instrumentation on aarch64
-        - os: ubuntu-24.04-arm
+        - os: ubuntu-22.04-arm
           bolt: true
     uses: ./.github/workflows/reusable-ubuntu.yml
     with:
-      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      config_hash: ${{ needs.build-context.outputs.config-hash }}
       bolt-optimizations: ${{ matrix.bolt }}
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
@@ -266,8 +264,8 @@ jobs:
     name: 'Ubuntu SSL tests with OpenSSL'
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       fail-fast: false
       matrix:
@@ -284,15 +282,15 @@ jobs:
       with:
         persist-credentials: false
     - name: Runner image version
-      run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
-    - name: Install Dependencies
+    - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars
       run: |
@@ -326,18 +324,18 @@ jobs:
 
   build_wasi:
     name: 'WASI'
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     uses: ./.github/workflows/reusable-wasi.yml
     with:
-      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      config_hash: ${{ needs.build-context.outputs.config-hash }}
 
   test_hypothesis:
     name: "Hypothesis tests on Ubuntu"
     runs-on: ubuntu-24.04
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     env:
       OPENSSL_VER: 3.0.15
       PYTHONSTRICTEXTENSIONBUILD: 1
@@ -347,7 +345,7 @@ jobs:
         persist-credentials: false
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
-    - name: Install Dependencies
+    - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Configure OpenSSL env vars
       run: |
@@ -379,12 +377,12 @@ jobs:
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR"
     - name: Runner image version
-      run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -452,8 +450,8 @@ jobs:
     name: 'Address sanitizer'
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       matrix:
         os: [ubuntu-24.04]
@@ -466,15 +464,15 @@ jobs:
       with:
         persist-credentials: false
     - name: Runner image version
-      run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
     - name: Restore config.cache
       uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
-    - name: Install Dependencies
+    - name: Install dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: Set up GCC-10 for ASAN
       uses: egor-tensin/setup-gcc@v1
@@ -515,41 +513,41 @@ jobs:
     name: >-
       Thread sanitizer
       ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     strategy:
       matrix:
         free-threading:
         - false
         - true
     uses: ./.github/workflows/reusable-tsan.yml
     with:
-      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      config_hash: ${{ needs.build-context.outputs.config-hash }}
       free-threading: ${{ matrix.free-threading }}
 
   cross-build-linux:
     name: Cross build Linux
     runs-on: ubuntu-latest
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
       - name: Runner image version
-        run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV"
+        run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
       - name: Restore config.cache
         uses: actions/cache@v4
         with:
           path: config.cache
-          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
+          key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
       - name: Register gcc problem matcher
         run: echo "::add-matcher::.github/problem-matchers/gcc.json"
       - name: Set build dir
         run:
           # an absolute path outside of the working directoy
           echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV"
-      - name: Install Dependencies
+      - name: Install dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Configure host build
         run: ./configure --prefix="$BUILD_DIR/host-python"
@@ -571,8 +569,8 @@ jobs:
     name: CIFuzz
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: check_source
-    if: needs.check_source.outputs.run_cifuzz == 'true'
+    needs: build-context
+    if: needs.build-context.outputs.run-ci-fuzz == 'true'
     permissions:
       security-events: write
     strategy:
@@ -611,7 +609,7 @@ jobs:
     if: always()
 
     needs:
-    - check_source  # Transitive dependency, needed to access `run_tests` value
+    - build-context  # Transitive dependency, needed to access `run-tests` value
     - check-docs
     - check_autoconf_regen
     - check_generated_files
@@ -621,6 +619,7 @@ jobs:
     - build_wasi
     - build_windows
     - build_windows_msi
+    - cross-build-linux
     - test_hypothesis
     - build_asan
     - build_tsan
@@ -639,38 +638,39 @@ jobs:
           test_hypothesis,
         allowed-skips: >-
           ${{
-            !fromJSON(needs.check_source.outputs.run-docs)
+            !fromJSON(needs.build-context.outputs.run-docs)
             && '
             check-docs,
             '
             || ''
           }}
           ${{
-            needs.check_source.outputs.run_tests != 'true'
+            needs.build-context.outputs.run-tests != 'true'
             && '
             check_autoconf_regen,
             check_generated_files,
             build_macos,
             build_ubuntu,
             build_ubuntu_ssltests,
             build_wasi,
-            build_windows,
             build_asan,
             build_tsan,
+            test_hypothesis,
+            cross-build-linux,
             '
             || ''
           }}
           ${{
-            !fromJSON(needs.check_source.outputs.run_cifuzz)
+            !fromJSON(needs.build-context.outputs.run-windows-tests)
             && '
-            cifuzz,
+            build_windows,
             '
             || ''
           }}
           ${{
-            !fromJSON(needs.check_source.outputs.run_hypothesis)
+            !fromJSON(needs.build-context.outputs.run-ci-fuzz)
             && '
-            test_hypothesis,
+            cifuzz,
             '
             || ''
           }}