Merge branch 'main' into issue-116860

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/fedora:37
+FROM docker.io/library/fedora:40
 
 ENV CC=clang
 

.github/CODEOWNERS

@@ -29,19 +29,21 @@ Objects/type* @markshannon
 Objects/codeobject.c @markshannon
 Objects/frameobject.c @markshannon
 Objects/call.c @markshannon
-Python/ceval*.c @markshannon @gvanrossum
-Python/ceval*.h @markshannon @gvanrossum
+Python/ceval*.c @markshannon
+Python/ceval*.h @markshannon
 Python/compile.c @markshannon @iritkatriel
 Python/assemble.c @markshannon @iritkatriel
 Python/flowgraph.c @markshannon @iritkatriel
 Python/ast_opt.c @isidentical
-Python/bytecodes.c @markshannon @gvanrossum
-Python/optimizer*.c @markshannon @gvanrossum
+Python/bytecodes.c @markshannon
+Python/optimizer*.c @markshannon
 Python/optimizer_analysis.c @Fidget-Spinner
 Python/optimizer_bytecodes.c @Fidget-Spinner
+Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv
 Lib/test/test_patma.py @brandtbucher
 Lib/test/test_type_*.py @JelleZijlstra
-Lib/test/test_capi/test_misc.py @markshannon @gvanrossum
+Lib/test/test_capi/test_misc.py @markshannon
+Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv
 Tools/c-analyzer/ @ericsnowcurrently
 
 # dbm
@@ -119,7 +121,7 @@ Python/dynload_*.c @ericsnowcurrently
 Lib/test/test_module/ @ericsnowcurrently
 Doc/c-api/module.rst @ericsnowcurrently
 **/*importlib/resources/* @jaraco @warsaw @FFY00
-**/importlib/metadata/* @jaraco @warsaw
+**/*importlib/metadata/* @jaraco @warsaw
 
 # Dates and times
 **/*datetime* @pganssle @abalkin
@@ -150,7 +152,7 @@ Include/internal/pycore_time.h @pganssle @abalkin
 /Lib/test/test_tokenize.py @pablogsal @lysnikolaou
 
 # Code generator
-/Tools/cases_generator/ @gvanrossum
+/Tools/cases_generator/ @markshannon
 
 # AST
 Python/ast.c @isidentical
@@ -243,6 +245,18 @@ Lib/test/support/interpreters/ @ericsnowcurrently
 Modules/_xx*interp*module.c @ericsnowcurrently
 Lib/test/test_interpreters/ @ericsnowcurrently
 
+# Android
+**/*Android* @mhsmith
+**/*android* @mhsmith
+
+# iOS (but not termios)
+**/iOS* @freakboy3742
+**/ios* @freakboy3742
+**/*_iOS* @freakboy3742
+**/*_ios* @freakboy3742
+**/*-iOS* @freakboy3742
+**/*-ios* @freakboy3742
+
 # WebAssembly
 /Tools/wasm/ @brettcannon
 

.github/workflows/build.yml

@@ -8,19 +8,11 @@ on:
  push:
  branches:
  - 'main'
- - '3.12'
- - '3.11'
- - '3.10'
- - '3.9'
- - '3.8'
+ - '3.*'
  pull_request:
  branches:
  - 'main'
- - '3.12'
- - '3.11'
- - '3.10'
- - '3.9'
- - '3.8'
+ - '3.*'
 
 permissions:
  contents: read
@@ -97,7 +89,7 @@ jobs:
  - name: Get a list of the changed documentation-related files
  if: github.event_name == 'pull_request'
  id: changed-docs-files
- uses: Ana06/get-changed-files@v2.2.0
+ uses: Ana06/get-changed-files@v2.3.0
  with:
  filter: |
  Doc/**
@@ -137,6 +129,7 @@ jobs:
  uses: actions/cache@v4
  with:
  path: config.cache
+ # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
  key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
  - name: Install Dependencies
  run: sudo ./.github/workflows/posix-deps-apt.sh
@@ -217,7 +210,7 @@ jobs:
  with:
  config_hash: ${{ needs.check_source.outputs.config_hash }}
  free-threading: true
- # macos-14 is M1
+ # macos-14-large is Intel with 12 cores (most parallelism)
  os-matrix: '["macos-14"]'
 
  build_ubuntu:
@@ -249,7 +242,7 @@ jobs:
 
  build_ubuntu_ssltests:
  name: 'Ubuntu SSL tests with OpenSSL'
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
  timeout-minutes: 60
  needs: check_source
  if: needs.check_source.outputs.run_tests == 'true'
@@ -315,7 +308,7 @@ jobs:
 
  test_hypothesis:
  name: "Hypothesis tests on Ubuntu"
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
  timeout-minutes: 60
  needs: check_source
  if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
@@ -428,7 +421,7 @@ jobs:
 
  build_asan:
  name: 'Address sanitizer'
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
  timeout-minutes: 60
  needs: check_source
  if: needs.check_source.outputs.run_tests == 'true'
@@ -484,6 +477,28 @@ jobs:
  - name: Tests
  run: xvfb-run make test
 
+ build_tsan:
+ name: 'Thread sanitizer'
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ uses: ./.github/workflows/reusable-tsan.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+ options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
+ suppressions_path: Tools/tsan/supressions.txt
+ tsan_logs_artifact_name: tsan-logs-default
+
+ build_tsan_free_threading:
+ name: 'Thread sanitizer (free-threading)'
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ uses: ./.github/workflows/reusable-tsan.yml
+ with:
+ config_hash: ${{ needs.check_source.outputs.config_hash }}
+ options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug
+ suppressions_path: Tools/tsan/suppressions_free_threading.txt
+ tsan_logs_artifact_name: tsan-logs-free-threading
+
  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
  cifuzz:
  name: CIFuzz
@@ -542,6 +557,8 @@ jobs:
  - build_windows_free_threading
  - test_hypothesis
  - build_asan
+ - build_tsan
+ - build_tsan_free_threading
  - cifuzz
 
  runs-on: ubuntu-latest
@@ -575,6 +592,8 @@ jobs:
  build_windows,
  build_windows_free_threading,
  build_asan,
+ build_tsan,
+ build_tsan_free_threading,
  '
  || ''
  }}

.github/workflows/jit.yml

@@ -5,24 +5,45 @@ on:
  - '**jit**'
  - 'Python/bytecodes.c'
  - 'Python/optimizer*.c'
- - 'Python/optimizer_bytecodes.c'
+ - '!Python/perf_jit_trampoline.c'
+ - '!**/*.md'
+ - '!**/*.ini'
  push:
  paths:
  - '**jit**'
  - 'Python/bytecodes.c'
  - 'Python/optimizer*.c'
- - 'Python/optimizer_bytecodes.c'
+ - '!Python/perf_jit_trampoline.c'
+ - '!**/*.md'
+ - '!**/*.ini'
  workflow_dispatch:
 
+permissions:
+ contents: read
+
 concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true
 
 jobs:
+ interpreter:
+ name: Interpreter (Debug)
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v4
+ - name: Build tier two interpreter
+ run: |
+ ./configure --enable-experimental-jit=interpreter --with-pydebug
+ make all --jobs 4
+ - name: Test tier two interpreter
+ run: |
+ ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
  jit:
  name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+ needs: interpreter
  runs-on: ${{ matrix.runner }}
- timeout-minutes: 60
+ timeout-minutes: 90
  strategy:
  fail-fast: false
  matrix:
@@ -40,7 +61,7 @@ jobs:
  - true
  - false
  llvm:
- - 16
+ - 18
  include:
  - target: i686-pc-windows-msvc/msvc
  architecture: Win32
@@ -74,14 +95,10 @@ jobs:
  architecture: aarch64
  runner: ubuntu-latest
  compiler: gcc
- # These fail because of emulation, not because of the JIT:
- exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
  - target: aarch64-unknown-linux-gnu/clang
  architecture: aarch64
  runner: ubuntu-latest
  compiler: clang
- # These fail because of emulation, not because of the JIT:
- exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
  env:
  CC: ${{ matrix.compiler }}
  steps:
@@ -93,37 +110,43 @@ jobs:
  - name: Native Windows
  if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
  run: |
+ choco upgrade llvm -y
  choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
  ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
- ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+ ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
  # No PGO or tests (yet):
  - name: Emulated Windows
  if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
  run: |
+ choco upgrade llvm -y
  choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
  ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
  - name: Native macOS
  if: runner.os == 'macOS'
  run: |
+ brew update
  brew install llvm@${{ matrix.llvm }}
- export SDKROOT="$(xcrun --show-sdk-path)"
- ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+ SDKROOT="$(xcrun --show-sdk-path)" \
+  ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
  make all --jobs 4
- ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+ ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+ # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
  - name: Native Linux
  if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
  run: |
  sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
  export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
- ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
  make all --jobs 4
- ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+ ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+ # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
  - name: Emulated Linux
  if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+ # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
  run: |
  sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
  export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
@@ -134,10 +157,29 @@ jobs:
  sudo apt install --yes "gcc-$HOST" qemu-user
  ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
  ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
- export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
- export CPP="$CC --preprocess"
- export HOSTRUNNER=qemu-${{ matrix.architecture }}
  export QEMU_LD_PREFIX="/usr/$HOST"
- ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+ CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
+ CPP="$CC --preprocess" \
+ HOSTRUNNER=qemu-${{ matrix.architecture }} \
+ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
  make all --jobs 4
- ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+ ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+ jit-with-disabled-gil:
+ name: Free-Threaded (Debug)
+ needs: interpreter
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ - name: Build with JIT enabled and GIL disabled
+ run: |
+ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18
+ export PATH="$(llvm-config-18 --bindir):$PATH"
+ ./configure --enable-experimental-jit --with-pydebug --disable-gil
+ make all --jobs 4
+ - name: Run tests
+ run: |
+ ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/lint.yml

@@ -23,4 +23,4 @@ jobs:
  - uses: actions/setup-python@v5
  with:
  python-version: "3.x"
- - uses: pre-commit/action@v3.0.0
+ - uses: pre-commit/action@v3.0.1

.github/workflows/mypy.yml

@@ -8,6 +8,7 @@ on:
  pull_request:
  paths:
  - ".github/workflows/mypy.yml"
+ - "Lib/_pyrepl/**"
  - "Lib/test/libregrtest/**"
  - "Tools/build/generate_sbom.py"
  - "Tools/cases_generator/**"
@@ -33,10 +34,12 @@ concurrency:
 jobs:
  mypy:
  strategy:
+ fail-fast: false
  matrix:
  target: [
+ "Lib/_pyrepl",
  "Lib/test/libregrtest",
- "Tools/build/",
+ "Tools/build",
  "Tools/cases_generator",
  "Tools/clinic",
  "Tools/jit",

.github/workflows/posix-deps-apt.sh

@@ -15,6 +15,7 @@ apt-get -yq install \
  libgdbm-dev \
  libgdbm-compat-dev \
  liblzma-dev \
+ libmpdec-dev \
  libncurses5-dev \
  libreadline6-dev \
  libsqlite3-dev \

.github/workflows/project-updater.yml

@@ -23,7 +23,7 @@ jobs:
  - { project: 32, label: sprint }
 
  steps:
- - uses: actions/add-to-project@v0.1.0
+ - uses: actions/add-to-project@v1.0.0
  with:
  project-url: https://github.com/orgs/python/projects/${{ matrix.project }}
  github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}