diff --git a/.editorconfig b/.editorconfig index 5b04b32a89e3d2..25bc5935258bd1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h}] +[*.{py,c,cpp,h,gram}] indent_size = 4 [*.rst] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 08d7a80d7726d3..02a7b5d45b4627 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -26,7 +26,8 @@ Modules/Setup* @erlend-aasland **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 -**/*jit* @brandtbucher @savannahostrowski +**/*jit* @brandtbucher @savannahostrowski @diegorusso +Python/perf_jit_trampoline.c # Exclude the owners of "**/*jit*", above. Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6171571857af6..f07f5e8040acf0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -260,7 +260,7 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 @@ -322,6 +322,81 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v4 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + build-wasi: name: 'WASI' needs: build-context @@ -510,31 +585,31 @@ jobs: - name: Tests run: xvfb-run make ci - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + build-san: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Sanitizers${{ '' }} needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: + check-name: + - Thread free-threading: - false - true - uses: ./.github/workflows/reusable-tsan.yml + sanitizer: + - TSan + include: + - check-name: Undefined behavior + sanitizer: UBSan + free-threading: false + uses: ./.github/workflows/reusable-san.yml with: + sanitizer: ${{ matrix.sanitizer }} config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} - build-ubsan: - name: Undefined behavior sanitizer - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - uses: ./.github/workflows/reusable-ubsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest @@ -628,11 +703,12 @@ jobs: - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl - build-wasi - test-hypothesis - build-asan - - build-tsan + - build-san - cross-build-linux - cifuzz if: always() @@ -643,7 +719,8 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, test-hypothesis, cifuzz, allowed-skips: >- @@ -661,11 +738,12 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, build-wasi, test-hypothesis, build-asan, - build-tsan, + build-san, cross-build-linux, ' || '' diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 116e0c1e945e38..947badff816c09 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,6 +5,8 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -13,6 +15,8 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 7773222af5d26f..0b64367e6c4562 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -5,6 +5,7 @@ apt-get -yq install \ build-essential \ pkg-config \ ccache \ + cmake \ gdb \ lcov \ libb2-dev \ @@ -25,3 +26,10 @@ apt-get -yq install \ uuid-dev \ xvfb \ zlib1g-dev + +# Workaround missing libmpdec-dev on ubuntu 24.04: +# https://launchpad.net/~ondrej/+archive/ubuntu/php +# https://deb.sury.org/ +sudo add-apt-repository ppa:ondrej/php +apt-get update +apt-get -yq install libmpdec-dev diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 657e0a6bf662f7..7b9dc4818577eb 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -66,7 +66,7 @@ jobs: run: | set -Eeuo pipefail # Build docs with the nit-picky option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | @@ -75,6 +75,18 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Build EPUB documentation' + continue-on-error: true + run: | + set -Eeuo pipefail + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub + pip install epubcheck + epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + if: github.event_name == 'pull_request' + continue-on-error: true # until gh-136155 is fixed + run: | + python Doc/tools/check-epub.py # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml new file mode 100644 index 00000000000000..e6ff02e4838ee6 --- /dev/null +++ b/.github/workflows/reusable-san.yml @@ -0,0 +1,124 @@ +name: Reusable Sanitizer + +on: + workflow_call: + inputs: + sanitizer: + required: true + type: string + config_hash: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build-san-reusable: + name: >- + ${{ inputs.sanitizer }}${{ + inputs.free-threading + && ' (free-threading)' + || '' + }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + + if [ "${SANITIZER}" = "TSan" ]; then + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSan crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + else + sudo ./llvm.sh 20 + fi + + - name: Sanitizer option setup + run: | + if [ "${SANITIZER}" = "TSan" ]; then + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + else + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + fi + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + env: + SANITIZER: ${{ inputs.sanitizer }} + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: >- + ./configure + --config-cache + ${{ + inputs.sanitizer == 'TSan' + && '--with-thread-sanitizer' + || '--with-undefined-behavior-sanitizer' + }} + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: >- + ./python -m test + ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + -j4 + - name: Parallel tests + if: >- + inputs.sanitizer == 'TSan' + && fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + - name: Display logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: >- + ${{ inputs.sanitizer }}-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: san_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml deleted file mode 100644 index 6a58e5305f8e09..00000000000000 --- a/.github/workflows/reusable-tsan.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Reusable Thread Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - free-threading: - description: Whether to use free-threaded mode - required: false - type: boolean - default: false - -env: - FORCE_COLOR: 1 - -jobs: - build-tsan-reusable: - name: 'Thread sanitizer' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - - name: Install dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-18 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN option setup - run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: >- - ./configure - --config-cache - --with-thread-sanitizer - --with-pydebug - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 - - name: Parallel tests - if: fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 - - name: Display TSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - - name: Archive TSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: >- - tsan-logs-${{ - fromJSON(inputs.free-threading) - && 'free-threading' - || 'default' - }} - path: tsan_log.* - if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubsan.yml b/.github/workflows/reusable-ubsan.yml deleted file mode 100644 index cf93932f13b827..00000000000000 --- a/.github/workflows/reusable-ubsan.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Reusable Undefined Behavior Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - -env: - FORCE_COLOR: 1 - -jobs: - build-ubsan-reusable: - name: 'Undefined behavior sanitizer' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - - name: Install dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-20 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 20 - - name: UBSAN option setup - run: | - echo "UBSAN_OPTIONS=halt_on_error=1:abort_on_error=1:print_summary=1:print_stacktrace=1" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: >- - ./configure - --config-cache - --with-undefined-behavior-sanitizer - --with-pydebug - - name: Set up UBSAN log after configuration - run: | - echo "UBSAN_OPTIONS=${UBSAN_OPTIONS}:log_path=${GITHUB_WORKSPACE}/ubsan_log" >> "$GITHUB_ENV" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test -j4 - - name: Display UBSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'ubsan_log.*' | xargs head -n 1000 - - name: Archive UBSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: >- - ubsan-logs - path: ubsan_log.* - if-no-files-found: ignore diff --git a/Android/android.py b/Android/android.py index 551168fc4b2f5a..75f73cd30993da 100755 --- a/Android/android.py +++ b/Android/android.py @@ -50,7 +50,19 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -118,7 +130,7 @@ def android_env(host): env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " - f"export HOST={host}; " + f"HOST={host}; " f"PREFIX={prefix}; " f". {env_script}; " f"export", @@ -175,7 +187,7 @@ def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1"]: + "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") shutil.unpack_archive(filename) @@ -453,17 +465,19 @@ async def logcat_task(context, initial_devices): # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + logcat_started = True level, message = match.groups() else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. + # If the regex doesn't match, this is either a logcat startup + # error, or the second or subsequent line of a multi-line + # message. Python won't produce multi-line messages, but other + # components might. level, message = None, line # Exclude high-volume messages which are rarely useful. @@ -483,25 +497,22 @@ async def logcat_task(context, initial_devices): # tag indicators from Python's stdout and stderr. for prefix in ["python.stdout: ", "python.stderr: "]: if message.startswith(prefix): - global logcat_started - logcat_started = True + global python_started + python_started = True stream.write(message.removeprefix(prefix)) break else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + log_verbose(context, line, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) def stop_app(serial): @@ -516,16 +527,6 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected - hidden_output = [] - - def log(line): - # Gradle may take several minutes to install SDK packages, so it's worth - # showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) - if context.command: mode = "-c" module = context.command @@ -550,7 +551,7 @@ def log(line): ] if context.verbose >= 2: args.append("--info") - log("> " + join_command(args)) + log_verbose(context, f"> {join_command(args)}\n") try: async with async_process( @@ -558,7 +559,12 @@ def log(line): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - log(line) + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -566,11 +572,6 @@ def log(line): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) @@ -600,6 +601,12 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] diff --git a/Doc/Makefile b/Doc/Makefile index c8a749a02a89ec..84578c5c57f478 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -170,6 +170,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ $(UV) venv --python=$(PYTHON) $(VENVDIR); \ diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index ab9f9c4539ae9a..3429a4eb652709 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` - structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -241,9 +237,11 @@ the Python object to the required type. For signed integer formats, :exc:`OverflowError` is raised if the value is out of range for the C type. -For unsigned integer formats, no range checking is done --- the +For unsigned integer formats, the most significant bits are silently truncated when the receiving field is too -small to receive the value. +small to receive the value, and :exc:`DeprecationWarning` is emitted when +the value is larger than the maximal value for the C type or less than +the minimal value for the corresponding signed integer type of the same size. ``b`` (:class:`int`) [unsigned char] Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C @@ -252,27 +250,25 @@ small to receive the value. ``B`` (:class:`int`) [unsigned char] Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. + Convert a Python integer to a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] Convert a Python integer to a C :c:expr:`short int`. ``H`` (:class:`int`) [unsigned short int] - Convert a Python integer to a C :c:expr:`unsigned short int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned short int`. ``i`` (:class:`int`) [int] Convert a Python integer to a plain C :c:expr:`int`. ``I`` (:class:`int`) [unsigned int] - Convert a Python integer to a C :c:expr:`unsigned int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned int`. ``l`` (:class:`int`) [long int] Convert a Python integer to a C :c:expr:`long int`. ``k`` (:class:`int`) [unsigned long] - Convert a Python integer to a C :c:expr:`unsigned long` without - overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long`. .. versionchanged:: 3.14 Use :meth:`~object.__index__` if available. @@ -281,8 +277,7 @@ small to receive the value. Convert a Python integer to a C :c:expr:`long long`. ``K`` (:class:`int`) [unsigned long long] - Convert a Python integer to a C :c:expr:`unsigned long long` - without overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long long`. .. versionchanged:: 3.14 Use :meth:`~object.__index__` if available. @@ -310,6 +305,14 @@ small to receive the value. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. +.. deprecated:: next + + For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``, + :exc:`DeprecationWarning` is emitted when the value is larger than + the maximal value for the C type or less than the minimal value for + the corresponding signed integer type of the same size. + + Other objects ------------- @@ -387,17 +390,6 @@ Other objects Non-tuple sequences are deprecated if *items* contains format units which store a borrowed buffer or a borrowed reference. -``unit?`` (anything or ``None``) [*matching-variable(s)*] - ``?`` modifies the behavior of the preceding format unit. - The C variable(s) corresponding to that parameter should be initialized - to their default value --- when the argument is ``None``, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding - C variable(s). - If the argument is not ``None``, it is parsed according to the specified - format unit. - - .. versionadded:: 3.14 - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 42594f063b0709..717b0da8f87c7f 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -211,6 +211,71 @@ bound into a function. .. versionadded:: 3.12 +.. _c_codeobject_flags: + +Code Object Flags +----------------- + +Code objects contain a bit-field of flags, which can be retrieved as the +:attr:`~codeobject.co_flags` Python attribute (for example using +:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to +:c:func:`PyUnstable_Code_New` and similar functions. + +Flags whose names start with ``CO_FUTURE_`` correspond to features normally +selectable by :ref:`future statements `. These flags can be used in +:c:member:`PyCompilerFlags.cf_flags`. +Note that many ``CO_FUTURE_`` flags are mandatory in current versions of +Python, and setting them has no effect. + +The following flags are available. +For their meaning, see the linked documentation of their Python equivalents. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Flag + * Meaning + * * .. c:macro:: CO_OPTIMIZED + * :py:data:`inspect.CO_OPTIMIZED` + * * .. c:macro:: CO_NEWLOCALS + * :py:data:`inspect.CO_NEWLOCALS` + * * .. c:macro:: CO_VARARGS + * :py:data:`inspect.CO_VARARGS` + * * .. c:macro:: CO_VARKEYWORDS + * :py:data:`inspect.CO_VARKEYWORDS` + * * .. c:macro:: CO_NESTED + * :py:data:`inspect.CO_NESTED` + * * .. c:macro:: CO_GENERATOR + * :py:data:`inspect.CO_GENERATOR` + * * .. c:macro:: CO_COROUTINE + * :py:data:`inspect.CO_COROUTINE` + * * .. c:macro:: CO_ITERABLE_COROUTINE + * :py:data:`inspect.CO_ITERABLE_COROUTINE` + * * .. c:macro:: CO_ASYNC_GENERATOR + * :py:data:`inspect.CO_ASYNC_GENERATOR` + * * .. c:macro:: CO_HAS_DOCSTRING + * :py:data:`inspect.CO_HAS_DOCSTRING` + * * .. c:macro:: CO_METHOD + * :py:data:`inspect.CO_METHOD` + + * * .. c:macro:: CO_FUTURE_DIVISION + * no effect (:py:data:`__future__.division`) + * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT + * no effect (:py:data:`__future__.absolute_import`) + * * .. c:macro:: CO_FUTURE_WITH_STATEMENT + * no effect (:py:data:`__future__.with_statement`) + * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION + * no effect (:py:data:`__future__.print_function`) + * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS + * no effect (:py:data:`__future__.unicode_literals`) + * * .. c:macro:: CO_FUTURE_GENERATOR_STOP + * no effect (:py:data:`__future__.generator_stop`) + * * .. c:macro:: CO_FUTURE_ANNOTATIONS + * :py:data:`__future__.annotations` + + Extra information ----------------- diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a750cda3e2d474..3ff4631a8e53c4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -982,184 +982,135 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. .. _standardexceptions: -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException (C var) - single: PyExc_BaseExceptionGroup (C var) - single: PyExc_Exception (C var) - single: PyExc_ArithmeticError (C var) - single: PyExc_AssertionError (C var) - single: PyExc_AttributeError (C var) - single: PyExc_BlockingIOError (C var) - single: PyExc_BrokenPipeError (C var) - single: PyExc_BufferError (C var) - single: PyExc_ChildProcessError (C var) - single: PyExc_ConnectionAbortedError (C var) - single: PyExc_ConnectionError (C var) - single: PyExc_ConnectionRefusedError (C var) - single: PyExc_ConnectionResetError (C var) - single: PyExc_EOFError (C var) - single: PyExc_FileExistsError (C var) - single: PyExc_FileNotFoundError (C var) - single: PyExc_FloatingPointError (C var) - single: PyExc_GeneratorExit (C var) - single: PyExc_ImportError (C var) - single: PyExc_IndentationError (C var) - single: PyExc_IndexError (C var) - single: PyExc_InterruptedError (C var) - single: PyExc_IsADirectoryError (C var) - single: PyExc_KeyError (C var) - single: PyExc_KeyboardInterrupt (C var) - single: PyExc_LookupError (C var) - single: PyExc_MemoryError (C var) - single: PyExc_ModuleNotFoundError (C var) - single: PyExc_NameError (C var) - single: PyExc_NotADirectoryError (C var) - single: PyExc_NotImplementedError (C var) - single: PyExc_OSError (C var) - single: PyExc_OverflowError (C var) - single: PyExc_PermissionError (C var) - single: PyExc_ProcessLookupError (C var) - single: PyExc_PythonFinalizationError (C var) - single: PyExc_RecursionError (C var) - single: PyExc_ReferenceError (C var) - single: PyExc_RuntimeError (C var) - single: PyExc_StopAsyncIteration (C var) - single: PyExc_StopIteration (C var) - single: PyExc_SyntaxError (C var) - single: PyExc_SystemError (C var) - single: PyExc_SystemExit (C var) - single: PyExc_TabError (C var) - single: PyExc_TimeoutError (C var) - single: PyExc_TypeError (C var) - single: PyExc_UnboundLocalError (C var) - single: PyExc_UnicodeDecodeError (C var) - single: PyExc_UnicodeEncodeError (C var) - single: PyExc_UnicodeError (C var) - single: PyExc_UnicodeTranslateError (C var) - single: PyExc_ValueError (C var) - single: PyExc_ZeroDivisionError (C var) - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BaseExceptionGroup` | :exc:`BaseExceptionGroup` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ +Exception and warning types +=========================== + +All standard Python exceptions and warning categories are available as global +variables whose names are ``PyExc_`` followed by the Python exception name. +These have the type :c:expr:`PyObject*`; they are all class objects. + +For completeness, here are all the variables: + +Exception types +--------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_BaseException + * :exc:`BaseException` + * * .. c:var:: PyObject *PyExc_BaseExceptionGroup + * :exc:`BaseExceptionGroup` + * * .. c:var:: PyObject *PyExc_Exception + * :exc:`Exception` + * * .. c:var:: PyObject *PyExc_ArithmeticError + * :exc:`ArithmeticError` + * * .. c:var:: PyObject *PyExc_AssertionError + * :exc:`AssertionError` + * * .. c:var:: PyObject *PyExc_AttributeError + * :exc:`AttributeError` + * * .. c:var:: PyObject *PyExc_BlockingIOError + * :exc:`BlockingIOError` + * * .. c:var:: PyObject *PyExc_BrokenPipeError + * :exc:`BrokenPipeError` + * * .. c:var:: PyObject *PyExc_BufferError + * :exc:`BufferError` + * * .. c:var:: PyObject *PyExc_ChildProcessError + * :exc:`ChildProcessError` + * * .. c:var:: PyObject *PyExc_ConnectionAbortedError + * :exc:`ConnectionAbortedError` + * * .. c:var:: PyObject *PyExc_ConnectionError + * :exc:`ConnectionError` + * * .. c:var:: PyObject *PyExc_ConnectionRefusedError + * :exc:`ConnectionRefusedError` + * * .. c:var:: PyObject *PyExc_ConnectionResetError + * :exc:`ConnectionResetError` + * * .. c:var:: PyObject *PyExc_EOFError + * :exc:`EOFError` + * * .. c:var:: PyObject *PyExc_FileExistsError + * :exc:`FileExistsError` + * * .. c:var:: PyObject *PyExc_FileNotFoundError + * :exc:`FileNotFoundError` + * * .. c:var:: PyObject *PyExc_FloatingPointError + * :exc:`FloatingPointError` + * * .. c:var:: PyObject *PyExc_GeneratorExit + * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportError + * :exc:`ImportError` + * * .. c:var:: PyObject *PyExc_IndentationError + * :exc:`IndentationError` + * * .. c:var:: PyObject *PyExc_IndexError + * :exc:`IndexError` + * * .. c:var:: PyObject *PyExc_InterruptedError + * :exc:`InterruptedError` + * * .. c:var:: PyObject *PyExc_IsADirectoryError + * :exc:`IsADirectoryError` + * * .. c:var:: PyObject *PyExc_KeyError + * :exc:`KeyError` + * * .. c:var:: PyObject *PyExc_KeyboardInterrupt + * :exc:`KeyboardInterrupt` + * * .. c:var:: PyObject *PyExc_LookupError + * :exc:`LookupError` + * * .. c:var:: PyObject *PyExc_MemoryError + * :exc:`MemoryError` + * * .. c:var:: PyObject *PyExc_ModuleNotFoundError + * :exc:`ModuleNotFoundError` + * * .. c:var:: PyObject *PyExc_NameError + * :exc:`NameError` + * * .. c:var:: PyObject *PyExc_NotADirectoryError + * :exc:`NotADirectoryError` + * * .. c:var:: PyObject *PyExc_NotImplementedError + * :exc:`NotImplementedError` + * * .. c:var:: PyObject *PyExc_OSError + * :exc:`OSError` + * * .. c:var:: PyObject *PyExc_OverflowError + * :exc:`OverflowError` + * * .. c:var:: PyObject *PyExc_PermissionError + * :exc:`PermissionError` + * * .. c:var:: PyObject *PyExc_ProcessLookupError + * :exc:`ProcessLookupError` + * * .. c:var:: PyObject *PyExc_PythonFinalizationError + * :exc:`PythonFinalizationError` + * * .. c:var:: PyObject *PyExc_RecursionError + * :exc:`RecursionError` + * * .. c:var:: PyObject *PyExc_ReferenceError + * :exc:`ReferenceError` + * * .. c:var:: PyObject *PyExc_RuntimeError + * :exc:`RuntimeError` + * * .. c:var:: PyObject *PyExc_StopAsyncIteration + * :exc:`StopAsyncIteration` + * * .. c:var:: PyObject *PyExc_StopIteration + * :exc:`StopIteration` + * * .. c:var:: PyObject *PyExc_SyntaxError + * :exc:`SyntaxError` + * * .. c:var:: PyObject *PyExc_SystemError + * :exc:`SystemError` + * * .. c:var:: PyObject *PyExc_SystemExit + * :exc:`SystemExit` + * * .. c:var:: PyObject *PyExc_TabError + * :exc:`TabError` + * * .. c:var:: PyObject *PyExc_TimeoutError + * :exc:`TimeoutError` + * * .. c:var:: PyObject *PyExc_TypeError + * :exc:`TypeError` + * * .. c:var:: PyObject *PyExc_UnboundLocalError + * :exc:`UnboundLocalError` + * * .. c:var:: PyObject *PyExc_UnicodeDecodeError + * :exc:`UnicodeDecodeError` + * * .. c:var:: PyObject *PyExc_UnicodeEncodeError + * :exc:`UnicodeEncodeError` + * * .. c:var:: PyObject *PyExc_UnicodeError + * :exc:`UnicodeError` + * * .. c:var:: PyObject *PyExc_UnicodeTranslateError + * :exc:`UnicodeTranslateError` + * * .. c:var:: PyObject *PyExc_ValueError + * :exc:`ValueError` + * * .. c:var:: PyObject *PyExc_ZeroDivisionError + * :exc:`ZeroDivisionError` .. versionadded:: 3.3 :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, @@ -1180,94 +1131,79 @@ the variables: .. versionadded:: 3.11 :c:data:`PyExc_BaseExceptionGroup`. -These are compatibility aliases to :c:data:`PyExc_OSError`: -.. index:: - single: PyExc_EnvironmentError (C var) - single: PyExc_IOError (C var) - single: PyExc_WindowsError (C var) +OSError aliases +--------------- -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`!PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ +The following are a compatibility aliases to :c:data:`PyExc_OSError`. .. versionchanged:: 3.3 These aliases used to be separate exception types. +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * Notes + * * .. c:var:: PyObject *PyExc_EnvironmentError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_IOError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_WindowsError + * :exc:`OSError` + * [win]_ + Notes: -.. [1] - This is a base class for other standard exceptions. +.. [win] + :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that + uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. .. _standardwarningcategories: -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning (C var) - single: PyExc_BytesWarning (C var) - single: PyExc_DeprecationWarning (C var) - single: PyExc_EncodingWarning (C var) - single: PyExc_FutureWarning (C var) - single: PyExc_ImportWarning (C var) - single: PyExc_PendingDeprecationWarning (C var) - single: PyExc_ResourceWarning (C var) - single: PyExc_RuntimeWarning (C var) - single: PyExc_SyntaxWarning (C var) - single: PyExc_UnicodeWarning (C var) - single: PyExc_UserWarning (C var) - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EncodingWarning` | :exc:`EncodingWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ +Warning types +------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_Warning + * :exc:`Warning` + * * .. c:var:: PyObject *PyExc_BytesWarning + * :exc:`BytesWarning` + * * .. c:var:: PyObject *PyExc_DeprecationWarning + * :exc:`DeprecationWarning` + * * .. c:var:: PyObject *PyExc_EncodingWarning + * :exc:`EncodingWarning` + * * .. c:var:: PyObject *PyExc_FutureWarning + * :exc:`FutureWarning` + * * .. c:var:: PyObject *PyExc_ImportWarning + * :exc:`ImportWarning` + * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning + * :exc:`PendingDeprecationWarning` + * * .. c:var:: PyObject *PyExc_ResourceWarning + * :exc:`ResourceWarning` + * * .. c:var:: PyObject *PyExc_RuntimeWarning + * :exc:`RuntimeWarning` + * * .. c:var:: PyObject *PyExc_SyntaxWarning + * :exc:`SyntaxWarning` + * * .. c:var:: PyObject *PyExc_UnicodeWarning + * :exc:`UnicodeWarning` + * * .. c:var:: PyObject *PyExc_UserWarning + * :exc:`UserWarning` .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. - -Notes: - -.. [3] - This is a base class for other standard warning categories. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index c5a7653efca26b..489676caa3a16a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -124,15 +124,15 @@ There are two problems on non-IEEE platforms: * What this does is undefined if *x* is a NaN or infinity. * ``-0.0`` and ``+0.0`` produce the same bytes string. -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack4(double x, char *p, int le) Pack a C double as the IEEE 754 binary32 single precision format. -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack8(double x, char *p, int le) Pack a C double as the IEEE 754 binary64 double precision format. @@ -154,14 +154,14 @@ Return value: The unpacked double. On error, this is ``-1.0`` and Note that on a non-IEEE platform this will refuse to unpack a bytes string that represents a NaN or infinity. -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack4(const char *p, int le) Unpack the IEEE 754 binary32 single precision format as a C double. -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack8(const char *p, int le) Unpack the IEEE 754 binary64 double precision format as a C double. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 41fd4ea14ef12f..e3ad4f4cdc52cc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -2277,6 +2277,18 @@ The C-API provides a basic mutual exclusion lock. .. versionadded:: 3.13 +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + .. _python-critical-section-api: Python Critical Section API @@ -2294,6 +2306,12 @@ is resumed, and its locks reacquired. This means the critical section API provides weaker guarantees than traditional locks -- they are useful because their behavior is similar to the :term:`GIL`. +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + The functions and structs used by the macros are exposed for cases where C macros are not available. They should only be used as in the given macro expansions. Note that the sizes and contents of the structures may @@ -2339,6 +2357,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION() Ends the critical section and releases the per-object lock. @@ -2368,6 +2403,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION2() Ends the critical section and releases the per-object locks. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 4fd10224262488..24be9ead3874d1 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -975,9 +975,7 @@ PyPreConfig Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option and the :envvar:`PYTHONUTF8` environment variable. - Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``. - - Default: ``-1`` in Python config and ``0`` in isolated config. + Default: ``1``. .. _c-preinit: diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 61fa49f8681cce..df1bb0ce370919 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -672,6 +672,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0fd159f1eb87f8..55f0d0f9fb7ff8 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -197,6 +197,13 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. + The function must not be called with ``NULL`` *v* and an an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + + .. versionchanged:: next + Must not be called with NULL value if an exception is set. + .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) @@ -207,6 +214,10 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The function must not be called with ``NULL`` *v* and an an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + The number of different attribute names passed to this function should be kept small, usually by using a statically allocated string as *attr_name*. @@ -215,6 +226,10 @@ Object Protocol For more details, see :c:func:`PyUnicode_InternFromString`, which may be used internally to create a key object. + .. versionchanged:: next + Must not be called with NULL value if an exception is set. + + .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) Generic attribute setter and deleter function that is meant @@ -596,12 +611,13 @@ Object Protocol if supported by the runtime. In the :term:`free-threaded ` build, this allows the interpreter to avoid reference count adjustments to *obj*, which may improve multi-threaded performance. The tradeoff is - that *obj* will only be deallocated by the tracing garbage collector. + that *obj* will only be deallocated by the tracing garbage collector, and + not when the interpreter no longer has any references to it. - This function returns ``1`` if deferred reference counting is enabled on *obj* - (including when it was enabled before the call), + This function returns ``1`` if deferred reference counting is enabled on *obj*, and ``0`` if deferred reference counting is not supported or if the hint was - ignored by the runtime. This function is thread-safe, and cannot fail. + ignored by the interpreter, such as when deferred reference counting is already + enabled on *obj*. This function is thread-safe, and cannot fail. This function does nothing on builds with the :term:`GIL` enabled, which do not support deferred reference counting. This also does nothing if *obj* is not @@ -609,7 +625,8 @@ Object Protocol :c:func:`PyObject_GC_IsTracked`). This function is intended to be used soon after *obj* is created, - by the code that creates it. + by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` + slot. .. versionadded:: 3.14 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 77b5e3c0876bbb..76a1e9f528dc70 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -5,11 +5,12 @@ Support for Perf Maps ---------------------- -On supported platforms (as of this writing, only Linux), the runtime can take +On supported platforms (Linux and macOS), the runtime can take advantage of *perf map files* to make Python functions visible to an external -profiling tool (such as `perf `_). -A running process may create a file in the ``/tmp`` directory, which contains entries -that can map a section of executable code to a name. This interface is described in the +profiling tool (such as `perf `_ or +`samply `_). A running process may create a +file in the ``/tmp`` directory, which contains entries that can map a section +of executable code to a name. This interface is described in the `documentation of the Linux Perf tool `_. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 987bc167c68d6c..58dd915e04f619 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -28,18 +28,52 @@ under :ref:`reference counting `. object. In a normal "release" build, it contains only the object's reference count and a pointer to the corresponding type object. Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + to a Python object can be cast to a :c:expr:`PyObject*`. + + The members must not be accessed directly; instead use macros such as + :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_refcnt + + The object's reference count, as returned by :c:macro:`Py_REFCNT`. + Do not use this field directly; instead use functions and macros such as + :c:macro:`!Py_REFCNT`, :c:func:`Py_INCREF` and :c:func:`Py_DecRef`. + + The field type may be different from ``Py_ssize_t``, depending on + build configuration and platform. + + .. c:member:: PyTypeObject* ob_type + + The object's type. + Do not use this field directly; use :c:macro:`Py_TYPE` and + :c:func:`Py_SET_TYPE` instead. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + An extension of :c:type:`PyObject` that adds the + :c:member:`~PyVarObject.ob_size` field. + This is intended for objects that have some notion of *length*. + + As with :c:type:`!PyObject`, the members must not be accessed directly; + instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and + :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_size + + A size field, whose contents should be considered an object's internal + implementation detail. + + Do not use this field directly; use :c:macro:`Py_SIZE` instead. + + Object creation functions such as :c:func:`PyObject_NewVar` will + generally set this field to the requested size (number of items). + After creation, arbitrary values can be stored in :c:member:`!ob_size` + using :c:macro:`Py_SET_SIZE`. + + To get an object's publicly exposed length, as returned by + the Python function :py:func:`len`, use :c:func:`PyObject_Length` + instead. .. c:macro:: PyObject_HEAD @@ -103,9 +137,8 @@ under :ref:`reference counting `. Get the type of the Python object *o*. - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. + The returned reference is :term:`borrowed ` from *o*. + Do not release it with :c:func:`Py_DECREF` or similar. .. versionchanged:: 3.11 :c:func:`Py_TYPE()` is changed to an inline static function. @@ -122,16 +155,26 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - Set the object *o* type to *type*. + Set the type of object *o* to *type*, without any checking or reference + counting. + + This is a very low-level operation. + Consider instead setting the Python attribute :attr:`~object.__class__` + using :c:func:`PyObject_SetAttrString` or similar. + + Note that assigning an incompatible type can lead to undefined behavior. + + If *type* is a :ref:`heap type `, the caller must create a + new reference to it. + Similarly, if the old type of *o* is a heap type, the caller must release + a reference to that type. .. versionadded:: 3.9 .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. + Get the :c:member:`~PyVarObject.ob_size` field of *o*. .. versionchanged:: 3.11 :c:func:`Py_SIZE()` is changed to an inline static function. @@ -140,7 +183,7 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size to *size*. + Set the :c:member:`~PyVarObject.ob_size` field of *o* to *size*. .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index af2bead3bb5004..060d6f60174b41 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -492,9 +492,9 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. -.. c:member:: Py_ssize_t PyObject.ob_refcnt +:c:member:`PyObject.ob_refcnt` - This is the type object's reference count, initialized to ``1`` by the + The type object's reference count is initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for @@ -506,7 +506,7 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is not inherited by subtypes. -.. c:member:: PyTypeObject* PyObject.ob_type +:c:member:`PyObject.ob_type` This is the type's type, in other words its metatype. It is initialized by the argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be @@ -532,14 +532,13 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. PyVarObject Slots ----------------- -.. c:member:: Py_ssize_t PyVarObject.ob_size +:c:member:`PyVarObject.ob_size` For :ref:`statically allocated type objects `, this should be initialized to zero. For :ref:`dynamically allocated type objects `, this field has a special internal meaning. - This field should be accessed using the :c:func:`Py_SIZE()` and - :c:func:`Py_SET_SIZE()` macros. + This field should be accessed using the :c:func:`Py_SIZE()` macro. **Inheritance:** diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 1ef4181d52eb10..fb07fec7effce8 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -361,7 +361,7 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. - .. c:var:: int CO_FUTURE_DIVISION - - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such + as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally + selectable using :ref:`future statements `. + See :ref:`c_codeobject_flags` for a complete list. diff --git a/Doc/conf.py b/Doc/conf.py index 161c2986441edd..1c1f36e5bc0737 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -233,75 +233,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BaseExceptionGroup'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_PythonFinalizationError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_EncodingWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), # Undocumented public C macros ('c:macro', 'Py_BUILD_ASSERT'), ('c:macro', 'Py_BUILD_ASSERT_EXPR'), @@ -517,6 +448,7 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +epub_exclude_files = ('index.xhtml', 'download.xhtml') # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 @@ -635,13 +567,14 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -ogp_custom_meta_tags = [ - '', -] -if 'create-social-cards' not in tags: # noqa: F821 - # Define a static preview image when not creating social cards - ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ - '', - '', +if 'builder_html' in tags: # noqa: F821 + ogp_custom_meta_tags = [ + '', ] + if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += [ + '', + '', + ] diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index b87f0a5ecde06f..a3e335ecaf4324 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -1,7 +1,6 @@ Pending removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdecimal``. * The :c:func:`!PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. * :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..9453f83799c43d --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.16 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdec``. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index a76d06cce1278a..9505bcfa05af11 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -45,7 +45,7 @@ Pending removal in Python 3.15 * :mod:`pathlib`: - * :meth:`.PurePath.is_reserved` + * :meth:`!.PurePath.is_reserved` has been deprecated since Python 3.13. Use :func:`os.path.isreserved` to detect reserved paths on Windows. @@ -97,6 +97,8 @@ Pending removal in Python 3.15 After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`wave`: * The ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 4c4a368baca955..edb672ed8ad9a2 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -89,8 +89,6 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - * :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in Python 3.12; use the *onexc* parameter instead. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index fd63495674651b..a89a69043c0f9f 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -214,7 +214,7 @@ and initialize it by calling :c:func:`PyErr_NewException` in the module's SpamError = PyErr_NewException("spam.error", NULL, NULL); -Since :c:data:`!SpamError` is a global variable, it will be overwitten every time +Since :c:data:`!SpamError` is a global variable, it will be overwritten every time the module is reinitialized, when the :c:data:`Py_mod_exec` function is called. For now, let's avoid the issue: we will block repeated initialization by raising an diff --git a/Doc/glossary.rst b/Doc/glossary.rst index c5c7994f1262a9..b7bd547d38fd1e 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -462,7 +462,7 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1208,6 +1208,11 @@ Glossary :func:`sys.getrefcount` function to return the reference count for a particular object. + In :term:`CPython`, reference counts are not considered to be stable + or well-defined values; the number of references to an object, and how + that number is affected by Python code, may be different between + versions. + regular package A traditional :term:`package`, such as a directory containing an ``__init__.py`` file. @@ -1280,6 +1285,16 @@ Glossary and ending with double underscores. Special methods are documented in :ref:`specialnames`. + standard library + The collection of :term:`packages `, :term:`modules ` + and :term:`extension modules ` distributed as a part + of the official Python interpreter package. The exact membership of the + collection may vary based on platform, available system libraries, or + other criteria. Documentation can be found at :ref:`library-index`. + + See also :data:`sys.stdlib_module_names` for a list of all possible + standard library module names. + statement A statement is part of a suite (a "block" of code). A statement is either an :term:`expression` or one of several constructs with a keyword, such @@ -1290,6 +1305,9 @@ Glossary issues such as incorrect types. See also :term:`type hints ` and the :mod:`typing` module. + stdlib + An abbreviation of :term:`standard library`. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1304,6 +1322,11 @@ Glossary See also :term:`borrowed reference`. + t-string + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 02b45879ccfaca..577e283bb9cb4c 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -161,6 +161,8 @@ that return :term:`strong references `. +===================================+===================================+ | :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyList_GET_ITEM` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 78e56e0c64fb10..053558e389030a 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1217,7 +1217,7 @@ flow inside a program. The book uses Scheme for its examples, but many of the design approaches described in these chapters are applicable to functional-style Python code. -https://www.defmacro.org/ramblings/fp.html: A general introduction to functional +https://defmacro.org/2006/06/19/fp.html: A general introduction to functional programming that uses Java examples and has a lengthy historical introduction. https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index fbc426ba1d7d9a..7da6dc8a39795e 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -626,8 +626,7 @@ Open Issues Several issues around per-module state and heap types are still open. -Discussions about improving the situation are best held on the `capi-sig -mailing list `__. +Discussions about improving the situation are best held on the `discuss forum under c-api tag `__. Per-Class Scope diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7d64a02358adb3..52537a91df542c 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4078,6 +4078,104 @@ lines. With this approach, you get better output: WARNING:demo: 1/0 WARNING:demo:ZeroDivisionError: division by zero +How to uniformly handle newlines in logging output +-------------------------------------------------- + +Usually, messages that are logged (say to console or file) consist of a single +line of text. However, sometimes there is a need to handle messages with +multiple lines - whether because a logging format string contains newlines, or +logged data contains newlines. If you want to handle such messages uniformly, so +that each line in the logged message appears uniformly formatted as if it was +logged separately, you can do this using a handler mixin, as in the following +snippet: + +.. code-block:: python + + # Assume this is in a module mymixins.py + import copy + + class MultilineMixin: + def emit(self, record): + s = record.getMessage() + if '\n' not in s: + super().emit(record) + else: + lines = s.splitlines() + rec = copy.copy(record) + rec.args = None + for line in lines: + rec.msg = line + super().emit(rec) + +You can use the mixin as in the following script: + +.. code-block:: python + + import logging + + from mymixins import MultilineMixin + + logger = logging.getLogger(__name__) + + class StreamHandler(MultilineMixin, logging.StreamHandler): + pass + + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-9s %(message)s', + handlers = [StreamHandler()]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +The script, when run, prints something like: + +.. code-block:: text + + 2025-07-02 13:54:47,234 DEBUG Single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me once ... + 2025-07-02 13:54:47,234 DEBUG Another single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me ... + 2025-07-02 13:54:47,234 DEBUG can't get fooled again + +If, on the other hand, you are concerned about `log injection +`_, you can use a +formatter which escapes newlines, as per the following example: + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + + class EscapingFormatter(logging.Formatter): + def format(self, record): + s = super().format(record) + return s.replace('\n', r'\n') + + if __name__ == '__main__': + h = logging.StreamHandler() + h.setFormatter(EscapingFormatter('%(asctime)s %(levelname)-9s %(message)s')) + logging.basicConfig(level=logging.DEBUG, handlers = [h]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +You can, of course, use whatever escaping scheme makes the most sense for you. +The script, when run, should produce output like this: + +.. code-block:: text + + 2025-07-09 06:47:33,783 DEBUG Single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me once ... + 2025-07-09 06:47:33,783 DEBUG Another single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me ...\ncan't get fooled again + +Escaping behaviour can't be the stdlib default , as it would break backwards +compatibility. .. patterns-to-avoid: diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 2982cf88bf97b4..b7225ff1c2cbfc 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -302,10 +302,10 @@ reading the following sections. If you're ready for that, grab some of your favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. +logging into your own scripts, and if you run into problems or don't understand +something, please post a question in the Help category of the `Python +discussion forum `_ and you should receive +help before too long. Still here? You can carry on reading the next few sections, which provide a slightly more advanced/in-depth tutorial than the basic one above. After that, diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 96d757ac452b5e..fc4772bbccab57 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -2,34 +2,35 @@ .. _perf_profiling: -============================================== -Python support for the Linux ``perf`` profiler -============================================== +======================================================== +Python support for the ``perf map`` compatible profilers +======================================================== :author: Pablo Galindo -`The Linux perf profiler `_ -is a very powerful tool that allows you to profile and obtain -information about the performance of your application. -``perf`` also has a very vibrant ecosystem of tools -that aid with the analysis of the data that it produces. +`The Linux perf profiler `_ and +`samply `_ are powerful tools that allow you to +profile and obtain information about the performance of your application. +Both tools have vibrant ecosystems that aid with the analysis of the data they produce. -The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only gets information about native symbols, that is, the names of +The main problem with using these profilers with Python applications is that +they only get information about native symbols, that is, the names of functions and procedures written in C. This means that the names and file names -of Python functions in your code will not appear in the output of ``perf``. +of Python functions in your code will not appear in the profiler output. Since Python 3.12, the interpreter can run in a special mode that allows Python -functions to appear in the output of the ``perf`` profiler. When this mode is +functions to appear in the output of compatible profilers. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the -fly before the execution of every Python function and it will teach ``perf`` the +fly before the execution of every Python function and it will teach the profiler the relationship between this piece of code and the associated Python function using :doc:`perf map files <../c-api/perfmaps>`. .. note:: - Support for the ``perf`` profiler is currently only available for Linux on - select architectures. Check the output of the ``configure`` build step or + Support for profiling is available on Linux and macOS on select architectures. + Perf is available on Linux, while samply can be used on both Linux and macOS. + samply support on macOS is available starting from Python 3.15. + Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -148,6 +149,31 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: +Using the samply profiler +------------------------- + +samply is a modern profiler that can be used as an alternative to perf. +It uses the same perf map files that Python generates, making it compatible +with Python's profiling support. samply is particularly useful on macOS +where perf is not available. + +To use samply with Python, first install it following the instructions at +https://github.com/mstange/samply, then run:: + + $ samply record PYTHONPERFSUPPORT=1 python my_script.py + +This will open a web interface where you can analyze the profiling data +interactively. The advantage of samply is that it provides a modern +web-based interface for analyzing profiling data and works on both Linux +and macOS. + +On macOS, samply support requires Python 3.15 or later. Also on macOS, samply +can't profile signed Python executables due to restrictions by macOS. You can +profile with Python binaries that you've compiled yourself, or which are +unsigned or locally-signed (such as anything installed by Homebrew). In +order to attach to running processes on macOS, run ``samply setup`` once (and +every time samply is updated) to self-sign the samply binary. + How to enable ``perf`` profiling support ---------------------------------------- diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index a46c1caefe4d8a..3a485a43a5a751 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -188,7 +188,7 @@ switch:: Once the Development & Deployment part of PPUG is fleshed out, some of those sections should be linked from new questions here (most notably, we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + https://packaging.python.org/en/latest/guides/index-mirrors-and-caches/) Common installation issues diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 4f3b663006fb28..5d916b30112d3c 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -37,38 +37,52 @@ No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evaluation of | -| | | | annotations using descriptors* | -+------------------+-------------+--------------+---------------------------------------------+ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * feature + * optional in + * mandatory in + * effect + * * .. data:: nested_scopes + * 2.1.0b1 + * 2.2 + * :pep:`227`: *Statically Nested Scopes* + * * .. data:: generators + * 2.2.0a1 + * 2.3 + * :pep:`255`: *Simple Generators* + * * .. data:: division + * 2.2.0a2 + * 3.0 + * :pep:`238`: *Changing the Division Operator* + * * .. data:: absolute_import + * 2.5.0a1 + * 3.0 + * :pep:`328`: *Imports: Multi-Line and Absolute/Relative* + * * .. data:: with_statement + * 2.5.0a1 + * 2.6 + * :pep:`343`: *The โ€œwithโ€ Statement* + * * .. data:: print_function + * 2.6.0a2 + * 3.0 + * :pep:`3105`: *Make print a function* + * * .. data:: unicode_literals + * 2.6.0a2 + * 3.0 + * :pep:`3112`: *Bytes literals in Python 3000* + * * .. data:: generator_stop + * 3.5.0b1 + * 3.7 + * :pep:`479`: *StopIteration handling inside generators* + * * .. data:: annotations + * 3.7.0b1 + * Never [1]_ + * :pep:`563`: *Postponed evaluation of annotations*, + :pep:`649`: *Deferred evaluation of annotations using descriptors* .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 7dfc11449a6cbc..981d89be7d58d6 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -511,7 +511,7 @@ code execution even with no access to any globals or builtins. For example: >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass ... - >>> annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) Hello world {'x': 'None'} diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index a03d88092dbf1f..79e15994491eff 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -74,7 +74,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=False) + *, suggest_on_error=False, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -119,7 +119,7 @@ ArgumentParser objects * suggest_on_error_ - Enables suggestions for mistyped argument choices and subparser names (default: ``False``) - * color_ - Allow color output (default: ``False``) + * color_ - Allow color output (default: ``True``) .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -434,12 +434,18 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Empty lines are treated as empty strings (``''``), which are allowed as values but + not as arguments. Empty lines that are read as arguments will result in an + "unrecognized arguments" error. + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -620,27 +626,19 @@ keyword argument:: color ^^^^^ -By default, the help message is printed in plain text. If you want to allow -color in help messages, you can enable it by setting ``color`` to ``True``:: +By default, the help message is printed in color using `ANSI escape sequences +`__. +If you want plain text help messages, you can disable this :ref:`in your local +environment `, or in the argument parser itself +by setting ``color`` to ``False``:: >>> parser = argparse.ArgumentParser(description='Process some integers.', - ... color=True) + ... color=False) >>> parser.add_argument('--action', choices=['sum', 'max']) >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) -Even if a CLI author has enabled color, it can be -:ref:`controlled using environment variables `. - -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``color`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.color = True - .. versionadded:: 3.14 @@ -839,23 +837,11 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -Only actions that consume command-line arguments (e.g. ``'store'``, -``'append'`` or ``'extend'``) can be used with positional arguments. - -.. class:: BooleanOptionalAction - - You may also specify an arbitrary action by passing an :class:`Action` subclass or - other object that implements the same interface. The :class:`!BooleanOptionalAction` - is available in :mod:`!argparse` and adds support for boolean actions such as - ``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - - .. versionadded:: 3.9 +You may also specify an arbitrary action by passing an :class:`Action` subclass +(e.g. :class:`BooleanOptionalAction`) or other object that implements the same +interface. Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'``, ``'extend'``, or custom actions with non-zero ``nargs``) can be used +with positional arguments. The recommended way to create a custom action is to extend :class:`Action`, overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and @@ -1429,6 +1415,21 @@ this API may be passed as the ``action`` parameter to and return a string which will be used when printing the usage of the program. If such method is not provided, a sensible default will be used. +.. class:: BooleanOptionalAction + + A subclass of :class:`Action` for handling boolean flags with positive + and negative options. Adding a single argument such as ``--foo`` automatically + creates both ``--foo`` and ``--no-foo`` options, storing ``True`` and ``False`` + respectively:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + + .. versionadded:: 3.9 + The parse_args() method ----------------------- diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ef6c62dca1e124..b24459b5c6346f 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -289,9 +289,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 115 (``ord('s')``): ``!s`` string formatting + * 114 (``ord('r')``): ``!r`` repr formatting + * 97 (``ord('a')``): ``!a`` ASCII formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -325,6 +325,54 @@ Literals Constant(value='.3')]))])) +.. class:: TemplateStr(values) + + A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name'), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place'), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + + .. versionadded:: 3.14 + + +.. class:: Interpolation(value, str, conversion, format_spec) + + Node representing a single interpolation field in a t-string. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``str`` is a constant containing the text of the interpolation expression. + * ``conversion`` is an integer: + + * -1: no conversion + * 115: ``!s`` string conversion + * 114: ``!r`` repr conversion + * 97: ``!a`` ascii conversion + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + .. versionadded:: 3.14 + + .. class:: List(elts, ctx) Tuple(elts, ctx) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d99213aa81d53e..d481a1921d532b 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -102,17 +102,34 @@ Queue .. method:: shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`QueueShutDown` in the formerly blocked thread. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will raise :exc:`QueueShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only - raise once the queue is empty. Set *immediate* to true to make - :meth:`~Queue.get` raise immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`QueueShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` - will be unblocked. If *immediate* is true, a task will be marked - as done for each remaining item in the queue, which may unblock - callers of :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 @@ -129,9 +146,6 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each - remaining item in the queue. - Raises :exc:`ValueError` if called more times than there were items placed in the queue. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 90f042aa377711..c7a3e0c596b9d0 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -192,12 +192,8 @@ The :mod:`bdb` module also defines two classes: entered. * ``"return"``: A function or other code block is about to return. * ``"exception"``: An exception has occurred. - * ``"c_call"``: A C function is about to be called. - * ``"c_return"``: A C function has returned. - * ``"c_exception"``: A C function has raised an exception. - For the Python events, specialized functions (see below) are called. For - the C events, no action is taken. + For all the events, specialized functions (see below) are called. The *arg* parameter depends on the previous event. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 86511602fa5a60..1cb0e225bca043 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -53,6 +53,14 @@ any codec: :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more information on codec error handling. +.. function:: charmap_build(string) + + Return a mapping suitable for encoding with a custom single-byte encoding. + Given a :class:`str` *string* of up to 256 characters representing a + decoding table, returns either a compact internal mapping object + ``EncodingMap`` or a :class:`dictionary ` mapping character ordinals + to byte values. Raises a :exc:`TypeError` on invalid input. + The full details for each codec can also be looked up directly: .. function:: lookup(encoding, /) @@ -235,8 +243,8 @@ wider range of codecs when working with binary files: .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`str` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental encoder. This function requires that the codec accept text :class:`str` objects @@ -247,8 +255,8 @@ wider range of codecs when working with binary files: .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`bytes` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental decoder. This function requires that the codec accept :class:`bytes` objects @@ -257,6 +265,20 @@ wider range of codecs when working with binary files: :func:`iterencode`. +.. function:: readbuffer_encode(buffer, errors=None, /) + + Return a :class:`tuple` containing the raw bytes of *buffer*, a + :ref:`buffer-compatible object ` or :class:`str` + (encoded to UTF-8 before processing), and their length in bytes. + + The *errors* argument is ignored. + + .. code-block:: pycon + + >>> codecs.readbuffer_encode(b"Zito") + (b'Zito', 4) + + The module also provides the following constants which are useful for reading and writing to platform dependent files: @@ -1043,8 +1065,15 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +case or use a hyphen instead of an underscore are also valid aliases +because they are equivalent when normalized by +:func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid +alias for the ``'utf_8'`` codec. + +.. note:: + + The below table lists the most common aliases, for a complete list + refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. But only codecs listed in the following table are guarantead to exist on @@ -1373,7 +1402,11 @@ encodings. | | | It is used in the Python | | | | pickle protocol. | +--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | +| undefined | | This Codec should only | +| | | be used for testing | +| | | purposes. | +| | | | +| | | Raise an exception for | | | | all conversions, even | | | | empty strings. The error | | | | handler is ignored. | @@ -1476,6 +1509,66 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. +:mod:`encodings` --- Encodings package +-------------------------------------- + +.. module:: encodings + :synopsis: Encodings package + +This module implements the following functions: + +.. function:: normalize_encoding(encoding) + + Normalize encoding name *encoding*. + + Normalization works as follows: all non-alphanumeric characters except the + dot used for Python package names are collapsed and replaced with a single + underscore, leading and trailing underscores are removed. + For example, ``' -;#'`` becomes ``'_'``. + + Note that *encoding* should be ASCII only. + + +.. note:: + The following functions should not be used directly, except for testing + purposes; :func:`codecs.lookup` should be used instead. + + +.. function:: search_function(encoding) + + Search for the codec module corresponding to the given encoding name + *encoding*. + + This function first normalizes the *encoding* using + :func:`normalize_encoding`, then looks for a corresponding alias. + It attempts to import a codec module from the encodings package using either + the alias or the normalized name. If the module is found and defines a valid + ``getregentry()`` function that returns a :class:`codecs.CodecInfo` object, + the codec is cached and returned. + + If the codec module defines a ``getaliases()`` function any returned aliases + are registered for future use. + + +.. function:: win32_code_page_search_function(encoding) + + Search for a Windows code page encoding *encoding* of the form ``cpXXXX``. + + If the code page is valid and supported, return a :class:`codecs.CodecInfo` + object for it. + + .. availability:: Windows. + + .. versionadded:: 3.14 + + +This module implements the following exception: + +.. exception:: CodecRegistryError + + Raised when a codec is invalid or incompatible. + + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 57ad8e3377fc67..a901403621b84f 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -523,8 +523,14 @@ Advanced parameter control .. attribute:: compression_level A high-level means of setting other compression parameters that affect - the speed and ratio of compressing data. Setting the level to zero uses - :attr:`COMPRESSION_LEVEL_DEFAULT`. + the speed and ratio of compressing data. + + Regular compression levels are greater than ``0``. Values greater than + ``20`` are considered "ultra" compression and require more memory than + other levels. Negative values can be used to trade off faster compression + for worse compression ratios. + + Setting the level to zero uses :attr:`COMPRESSION_LEVEL_DEFAULT`. .. attribute:: window_log diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 8860418e87a585..be9d565f8e0d38 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -9,21 +9,30 @@ .. versionadded:: 3.14 -**Source code:** :source:`Lib/concurrent/interpreters.py` +**Source code:** :source:`Lib/concurrent/interpreters` -------------- - -Introduction ------------- - The :mod:`!concurrent.interpreters` module constructs higher-level interfaces on top of the lower level :mod:`!_interpreters` module. -.. XXX Add references to the upcoming HOWTO docs in the seealso block. +The module is primarily meant to provide a basic API for managing +interpreters (AKA "subinterpreters") and running things in them. +Running mostly involves switching to an interpreter (in the current +thread) and calling a function in that execution context. + +For concurrency, interpreters themselves (and this module) don't +provide much more than isolation, which on its own isn't useful. +Actual concurrency is available separately through +:mod:`threads ` See `below `_ .. seealso:: + :class:`~concurrent.futures.InterpreterPoolExecutor` + combines threads with interpreters in a familiar interface. + + .. XXX Add references to the upcoming HOWTO docs in the seealso block. + :ref:`isolating-extensions-howto` how to update an extension module to support multiple interpreters @@ -41,18 +50,155 @@ interfaces on top of the lower level :mod:`!_interpreters` module. Key details ----------- -Before we dive into examples, there are a small number of details +Before we dive in further, there are a small number of details to keep in mind about using multiple interpreters: -* isolated, by default +* `isolated `_, by default * no implicit threads * not all PyPI packages support use in multiple interpreters yet .. XXX Are there other relevant details to list? -In the context of multiple interpreters, "isolated" means that -different interpreters do not share any state. In practice, there is some -process-global data they all share, but that is managed by the runtime. + +.. _interpreters-intro: + +Introduction +------------ + +An "interpreter" is effectively the execution context of the Python +runtime. It contains all of the state the runtime needs to execute +a program. This includes things like the import state and builtins. +(Each thread, even if there's only the main thread, has some extra +runtime state, in addition to the current interpreter, related to +the current exception and the bytecode eval loop.) + +The concept and functionality of the interpreter have been a part of +Python since version 2.2, but the feature was only available through +the C-API and not well known, and the `isolation `_ +was relatively incomplete until version 3.12. + +.. _interp-isolation: + +Multiple Interpreters and Isolation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A Python implementation may support using multiple interpreters in the +same process. CPython has this support. Each interpreter is +effectively isolated from the others (with a limited number of +carefully managed process-global exceptions to the rule). + +That isolation is primarily useful as a strong separation between +distinct logical components of a program, where you want to have +careful control of how those components interact. + +.. note:: + + Interpreters in the same process can technically never be strictly + isolated from one another since there are few restrictions on memory + access within the same process. The Python runtime makes a best + effort at isolation but extension modules may easily violate that. + Therefore, do not use multiple interpreters in security-sensitive + situations, where they shouldn't have access to each other's data. + +Running in an Interpreter +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running in a different interpreter involves switching to it in the +current thread and then calling some function. The runtime will +execute the function using the current interpreter's state. The +:mod:`!concurrent.interpreters` module provides a basic API for +creating and managing interpreters, as well as the switch-and-call +operation. + +No other threads are automatically started for the operation. +There is `a helper `_ for that though. +There is another dedicated helper for calling the builtin +:func:`exec` in an interpreter. + +When :func:`exec` (or :func:`eval`) are called in an interpreter, +they run using the interpreter's :mod:`!__main__` module as the +"globals" namespace. The same is true for functions that aren't +associated with any module. This is the same as how scripts invoked +from the command-line run in the :mod:`!__main__` module. + + +.. _interp-concurrency: + +Concurrency and Parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted earlier, interpreters do not provide any concurrency +on their own. They strictly represent the isolated execution +context the runtime will use *in the current thread*. That isolation +makes them similar to processes, but they still enjoy in-process +efficiency, like threads. + +All that said, interpreters do naturally support certain flavors of +concurrency, as a powerful side effect of that isolation. +There's a powerful side effect of that isolation. It enables a +different approach to concurrency than you can take with async or +threads. It's a similar concurrency model to CSP or the actor model, +a model which is relatively easy to reason about. + +You can take advantage of that concurrency model in a single thread, +switching back and forth between interpreters, Stackless-style. +However, this model is more useful when you combine interpreters +with multiple threads. This mostly involves starting a new thread, +where you switch to another interpreter and run what you want there. + +Each actual thread in Python, even if you're only running in the main +thread, has its own *current* execution context. Multiple threads can +use the same interpreter or different ones. + +At a high level, you can think of the combination of threads and +interpreters as threads with opt-in sharing. + +As a significant bonus, interpreters are sufficiently isolated that +they do not share the :term:`GIL`, which means combining threads with +multiple interpreters enables full multi-core parallelism. +(This has been the case since Python 3.12.) + +Communication Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In practice, multiple interpreters are useful only if we have a way +to communicate between them. This usually involves some form of +message passing, but can even mean sharing data in some carefully +managed way. + +With this in mind, the :mod:`!concurrent.interpreters` module provides +a :class:`queue.Queue` implementation, available through +:func:`create_queue`. + +.. _interp-object-sharing: + +"Sharing" Objects +^^^^^^^^^^^^^^^^^ + +Any data actually shared between interpreters loses the thread-safety +provided by the :term:`GIL`. There are various options for dealing with +this in extension modules. However, from Python code the lack of +thread-safety means objects can't actually be shared, with a few +exceptions. Instead, a copy must be created, which means mutable +objects won't stay in sync. + +By default, most objects are copied with :mod:`pickle` when they are +passed to another interpreter. Nearly all of the immutable builtin +objects are either directly shared or copied efficiently. For example: + +* :const:`None` +* :class:`bool` (:const:`True` and :const:`False`) +* :class:`bytes` +* :class:`str` +* :class:`int` +* :class:`float` +* :class:`tuple` (of similarly supported objects) + +There is a small number of Python types that actually share mutable +data between interpreters: + +* :class:`memoryview` +* :class:`Queue` Reference @@ -73,12 +219,19 @@ This module defines the following functions: .. function:: get_main() Return an :class:`Interpreter` object for the main interpreter. + This is the interpreter the runtime created to run the :term:`REPL` + or the script given at the command-line. It is usually the only one. .. function:: create() Initialize a new (idle) Python interpreter and return a :class:`Interpreter` object for it. +.. function:: create_queue() + + Initialize a new cross-interpreter queue and return a :class:`Queue` + object for it. + Interpreter objects ^^^^^^^^^^^^^^^^^^^ @@ -94,7 +247,7 @@ Interpreter objects (read-only) - The interpreter's ID. + The underlying interpreter's ID. .. attribute:: whence @@ -113,8 +266,10 @@ Interpreter objects .. method:: prepare_main(ns=None, **kwargs) - Bind "shareable" objects in the interpreter's - :mod:`!__main__` module. + Bind objects in the interpreter's :mod:`!__main__` module. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. .. method:: exec(code, /, dedent=True) @@ -125,6 +280,8 @@ Interpreter objects Return the result of calling running the given function in the interpreter (in the current thread). + .. _interp-call-in-thread: + .. method:: call_in_thread(callable, /, *args, **kwargs) Run the given function in the interpreter (in a new thread). @@ -159,7 +316,36 @@ Exceptions an object cannot be sent to another interpreter. -.. XXX Add functions for communicating between interpreters. +Communicating Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Queue(id) + + A wrapper around a low-level, cross-interpreter queue, which + implements the :class:`queue.Queue` interface. The underlying queue + can only be created through :func:`create_queue`. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. attribute:: id + + (read-only) + + The queue's ID. + + +.. exception:: QueueEmptyError + + This exception, a subclass of :exc:`queue.Empty`, is raised from + :meth:`!Queue.get` and :meth:`!Queue.get_nowait` when the queue + is empty. + +.. exception:: QueueFullError + + This exception, a subclass of :exc:`queue.Full`, is raised from + :meth:`!Queue.put` and :meth:`!Queue.put_nowait` when the queue + is full. Basic usage @@ -184,6 +370,12 @@ Creating an interpreter and running code in it:: print('spam!') """)) + def run(arg): + return arg + + res = interp.call(run, 'spam!') + print(res) + def run(): print('spam!') @@ -193,6 +385,3 @@ Creating an interpreter and running code in it:: t = interp.call_in_thread(run) t.join() - - -.. XXX Explain about object "sharing". diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 2e513bff651cd2..d39c4ca4a5838b 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -53,7 +53,7 @@ The :mod:`csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -84,7 +84,7 @@ The :mod:`csv` module defines the following functions: Spam, Lovely Spam, Wonderful Spam -.. function:: writer(csvfile, dialect='excel', **fmtparams) +.. function:: writer(csvfile, /, dialect='excel', **fmtparams) Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a @@ -323,8 +323,8 @@ The :mod:`csv` module defines the following constants: .. data:: QUOTE_MINIMAL Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. + special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator*. .. data:: QUOTE_NONNUMERIC @@ -342,10 +342,13 @@ The :mod:`csv` module defines the following constants: .. data:: QUOTE_NONE - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if + Instructs :class:`writer` objects to never quote fields. + When the current *delimiter*, *quotechar*, *escapechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator* occurs in output data + it is preceded by the current *escapechar* character. + If *escapechar* is not set, the writer will raise :exc:`Error` if any characters that require escaping are encountered. + Set *quotechar* to ``None`` to prevent its escaping. Instructs :class:`reader` objects to perform no special processing of quote characters. @@ -414,9 +417,16 @@ Dialects support the following attributes: .. attribute:: Dialect.escapechar - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from + A one-character string used by the writer to escape characters that + require escaping: + + * the *delimiter*, the *quotechar*, ``'\r'``, ``'\n'`` and any of the + characters in *lineterminator* are escaped if *quoting* is set to + :const:`QUOTE_NONE`; + * the *quotechar* is escaped if *doublequote* is :const:`False`; + * the *escapechar* itself. + + On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. .. versionchanged:: 3.11 @@ -436,9 +446,12 @@ Dialects support the following attributes: .. attribute:: Dialect.quotechar - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. + A one-character string used to quote fields containing special characters, + such as the *delimiter* or the *quotechar*, or which contain new-line + characters (``'\r'``, ``'\n'`` or any of the characters in *lineterminator*). + It defaults to ``'"'``. + Can be set to ``None`` to prevent escaping ``'"'`` if *quoting* is set + to :const:`QUOTE_NONE`. .. versionchanged:: 3.11 An empty *quotechar* is not allowed. @@ -447,7 +460,8 @@ Dialects support the following attributes: Controls when quotes should be generated by the writer and recognised by the reader. It can take on any of the :ref:`QUOTE_\* constants ` - and defaults to :const:`QUOTE_MINIMAL`. + and defaults to :const:`QUOTE_MINIMAL` if *quotechar* is not ``None``, + and :const:`QUOTE_NONE` otherwise. .. attribute:: Dialect.skipinitialspace diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 846cece3761858..09f596101b4d1e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -232,8 +232,24 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_int` | :c:expr:`int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_int8` | :c:type:`int8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int16` | :c:type:`int16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int32` | :c:type:`int32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int64` | :c:type:`int64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_uint` | :c:expr:`unsigned int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint8` | :c:type:`uint8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint16` | :c:type:`uint16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint32` | :c:type:`uint32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint64` | :c:type:`uint64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_long` | :c:expr:`long` | int | +----------------------+------------------------------------------+----------------------------+ | :class:`c_ulong` | :c:expr:`unsigned long` | int | @@ -2524,7 +2540,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2599,7 +2615,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 11685a32f48e4f..ac8a911c40a860 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1120,6 +1120,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` from a tuple + of strings and a tuple of interpolations and pushes the resulting instance + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` from a + value and its source expression and pushes the resulting instance onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index fb43cf918b84dd..5a2c6bdd27c386 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -311,9 +311,13 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. currentmodule:: None + .. attribute:: module.__test__ :no-typesetting: +.. currentmodule:: doctest + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 4285c436e8da80..5754c2b65b239f 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: :meth:`set_payload` instead. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the related ``make`` and ``add`` methods. @@ -224,7 +224,7 @@ Here are the methods of the :class:`Message` class: ASCII charset. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.get_content` and :meth:`~email.message.EmailMessage.iter_parts`. @@ -236,7 +236,7 @@ Here are the methods of the :class:`Message` class: the message's default character set; see :meth:`set_charset` for details. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content`. @@ -265,9 +265,9 @@ Here are the methods of the :class:`Message` class: using that :mailheader:`Content-Transfer-Encoding` and is not modified. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. + :meth:`email.message.EmailMessage.set_content` method. .. method:: get_charset() @@ -276,7 +276,7 @@ Here are the methods of the :class:`Message` class: message's payload. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns + :class:`~email.message.EmailMessage` class it always returns ``None``. @@ -486,7 +486,7 @@ Here are the methods of the :class:`Message` class: search instead of :mailheader:`Content-Type`. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -524,7 +524,7 @@ Here are the methods of the :class:`Message` class: to ``False``. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -579,7 +579,7 @@ Here are the methods of the :class:`Message` class: header is also added. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the ``make_`` and ``add_`` methods. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 439b5c8f34b65a..6a70714dc3ee42 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -116,7 +116,7 @@ Here is the API for the :class:`BytesFeedParser`: Works like :class:`BytesFeedParser` except that the input to the :meth:`~BytesFeedParser.feed` method must be a string. This is of limited utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is + contain only ASCII text or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, no binary attachments. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -159,7 +159,7 @@ message body, instead setting the payload to the raw body. methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) + (or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, :rfc:`6532`) style headers and header continuation lines, optionally preceded by an envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c9b2c7d76b6746..2cfc2f4962979f 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -504,16 +504,31 @@ Data Types .. class:: StrEnum - ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used - in most of the same places that a string can be used. The result of any string - operation performed on or with a *StrEnum* member is not part of the enumeration. + *StrEnum* is the same as :class:`Enum`, but its members are also strings and + can be used in most of the same places that a string can be used. The result + of any string operation performed on or with a *StrEnum* member is not part + of the enumeration. + + >>> from enum import StrEnum, auto + >>> class Color(StrEnum): + ... RED = 'r' + ... GREEN = 'g' + ... BLUE = 'b' + ... UNKNOWN = auto() + ... + >>> Color.RED + + >>> Color.UNKNOWN + + >>> str(Color.UNKNOWN) + 'unknown' .. note:: There are places in the stdlib that check for an exact :class:`str` instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` instead of ``isinstance(unknown, str)``), and in those locations you - will need to use ``str(StrEnum.member)``. + will need to use ``str(MyStrEnum.MY_MEMBER)``. .. note:: diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 9806ae80905ca0..a73cc2e8a2dd3e 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -204,10 +204,16 @@ The following exceptions are the exceptions that are usually raised. assignment fails. (When an object does not support attribute references or attribute assignments at all, :exc:`TypeError` is raised.) - The :attr:`name` and :attr:`obj` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the attribute - that was attempted to be accessed and the object that was accessed for said - attribute, respectively. + The optional *name* and *obj* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the attribute that was attempted to be accessed. + + .. attribute:: obj + + The object that was accessed for the named attribute. .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. @@ -215,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and + without reading any data. (Note: the :meth:`!io.IOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -312,9 +318,11 @@ The following exceptions are the exceptions that are usually raised. unqualified names. The associated value is an error message that includes the name that could not be found. - The :attr:`name` attribute can be set using a keyword-only argument to the - constructor. When set it represent the name of the variable that was attempted - to be accessed. + The optional *name* keyword-only argument sets the attribute: + + .. attribute:: name + + The name of the variable that was attempted to be accessed. .. versionchanged:: 3.10 Added the :attr:`name` attribute. @@ -382,7 +390,7 @@ The following exceptions are the exceptions that are usually raised. The corresponding error message, as provided by the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` + functions :c:func:`!perror` under POSIX, and :c:func:`!FormatMessage` under Windows. .. attribute:: filename @@ -398,7 +406,7 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the + :exc:`!mmap.error` have been merged into :exc:`OSError`, and the constructor may return a subclass. .. versionchanged:: 3.4 @@ -429,7 +437,9 @@ The following exceptions are the exceptions that are usually raised. * Creating a new Python thread. * :meth:`Joining ` a running daemon thread. - * :func:`os.fork`. + * :func:`os.fork`, + * acquiring a lock such as :class:`threading.Lock`, when it is known that + the operation would otherwise deadlock. See also the :func:`sys.is_finalizing` function. @@ -440,6 +450,11 @@ The following exceptions are the exceptions that are usually raised. :meth:`threading.Thread.join` can now raise this exception. + .. versionchanged:: next + + This exception may be raised when acquiring :meth:`threading.Lock` + or :meth:`threading.RLock`. + .. exception:: RecursionError This exception is derived from :exc:`RuntimeError`. It is raised when the @@ -590,7 +605,7 @@ The following exceptions are the exceptions that are usually raised. handled, the Python interpreter exits; no stack traceback is printed. The constructor accepts the same optional argument passed to :func:`sys.exit`. If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if + C's :c:func:`!exit` function); if it is ``None``, the exit status is zero; if it has another type (such as a string), the object's value is printed and the exit status is one. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 1977f4d3ba3916..677966a8b2eaab 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -228,6 +228,41 @@ handler: Fatal Python error: Segmentation fault Current thread 0x00007fb899f39700 (most recent call first): - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at + File "/opt/python/Lib/ctypes/__init__.py", line 486 in string_at File "", line 1 in + + Current thread's C stack trace (most recent call first): + Binary file "/opt/python/python", at _Py_DumpStack+0x42 [0x5b27f7d7147e] + Binary file "/opt/python/python", at +0x32dcbd [0x5b27f7d85cbd] + Binary file "/opt/python/python", at +0x32df8a [0x5b27f7d85f8a] + Binary file "/usr/lib/libc.so.6", at +0x3def0 [0x77b73226bef0] + Binary file "/usr/lib/libc.so.6", at +0x17ef9c [0x77b7323acf9c] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0xcdf6 [0x77b7315dddf6] + Binary file "/usr/lib/libffi.so.8", at +0x7976 [0x77b73158f976] + Binary file "/usr/lib/libffi.so.8", at +0x413c [0x77b73158c13c] + Binary file "/usr/lib/libffi.so.8", at ffi_call+0x12e [0x77b73158ef0e] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0x15a33 [0x77b7315e6a33] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0x164fa [0x77b7315e74fa] + Binary file "/opt/python/build/lib.linux-x86_64-3.15/_ctypes.cpython-315d-x86_64-linux-gnu.so", at +0xc624 [0x77b7315dd624] + Binary file "/opt/python/python", at _PyObject_MakeTpCall+0xce [0x5b27f7b73883] + Binary file "/opt/python/python", at +0x11bab6 [0x5b27f7b73ab6] + Binary file "/opt/python/python", at PyObject_Vectorcall+0x23 [0x5b27f7b73b04] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x490c [0x5b27f7cbb302] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at PyEval_EvalCode+0xc5 [0x5b27f7cd9ba3] + Binary file "/opt/python/python", at +0x255957 [0x5b27f7cad957] + Binary file "/opt/python/python", at +0x255ab4 [0x5b27f7cadab4] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x6c3e [0x5b27f7cbd634] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at +0x11b6e1 [0x5b27f7b736e1] + Binary file "/opt/python/python", at +0x11d348 [0x5b27f7b75348] + Binary file "/opt/python/python", at +0x11d626 [0x5b27f7b75626] + Binary file "/opt/python/python", at PyObject_Call+0x20 [0x5b27f7b7565e] + Binary file "/opt/python/python", at +0x32a67a [0x5b27f7d8267a] + Binary file "/opt/python/python", at +0x32a7f8 [0x5b27f7d827f8] + Binary file "/opt/python/python", at +0x32ac1b [0x5b27f7d82c1b] + Binary file "/opt/python/python", at Py_RunMain+0x31 [0x5b27f7d82ebe] + Segmentation fault diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 12e61bc36f5db0..ee654b7a83e203 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -53,7 +53,7 @@ a :class:`!str` filename, and vice-versa. Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following -functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. .. function:: fnmatch(name, pat) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index fc7f9a6301a915..8796056b4b8722 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. + with value equal to ``numerator/denominator`` where the denominator is positive. + If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -142,7 +142,7 @@ another rational number, or from a string. .. versionadded:: 3.12 - .. classmethod:: from_float(flt) + .. classmethod:: from_float(f) Alternative constructor which only accepts instances of :class:`float` or :class:`numbers.Integral`. Beware that diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3e75621be6dad3..beec9b942afc0f 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -199,12 +199,18 @@ The :mod:`functools` module defines the following functions: and *typed*. This is for information purposes only. Mutating the values has no effect. + .. method:: lru_cache.cache_info() + :no-typesetting: + To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` + parameter, the wrapped function is instrumented with a :func:`!cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, *maxsize* and *currsize*. - The decorator also provides a :func:`cache_clear` function for clearing or + .. method:: lru_cache.cache_clear() + :no-typesetting: + + The decorator also provides a :func:`!cache_clear` function for clearing or invalidating the cache. The original underlying function is accessible through the @@ -284,9 +290,9 @@ The :mod:`functools` module defines the following functions: class decorator supplies the rest. This simplifies the effort involved in specifying all of the possible rich comparison operations: - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. + The class must define one of :meth:`~object.__lt__`, :meth:`~object.__le__`, + :meth:`~object.__gt__`, or :meth:`~object.__ge__`. + In addition, the class should supply an :meth:`~object.__eq__` method. For example:: @@ -418,7 +424,7 @@ The :mod:`functools` module defines the following functions: like normal functions, are handled as descriptors). When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or + :func:`classmethod`, :func:`staticmethod`, :func:`~abc.abstractmethod` or another instance of :class:`partialmethod`), calls to ``__get__`` are delegated to the underlying descriptor, and an appropriate :ref:`partial object` returned as the result. @@ -499,7 +505,10 @@ The :mod:`functools` module defines the following functions: ... print("Let me just say,", end=" ") ... print(arg) - To add overloaded implementations to the function, use the :func:`register` + .. method:: singledispatch.register() + :no-typesetting: + + To add overloaded implementations to the function, use the :func:`!register` attribute of the generic function, which can be used as a decorator. For functions annotated with types, the decorator will infer the type of the first argument automatically:: @@ -565,14 +574,14 @@ The :mod:`functools` module defines the following functions: runtime impact. To enable registering :term:`lambdas` and pre-existing functions, - the :func:`register` attribute can also be used in a functional form:: + the :func:`~singledispatch.register` attribute can also be used in a functional form:: >>> def nothing(arg, verbose=False): ... print("Nothing.") ... >>> fun.register(type(None), nothing) - The :func:`register` attribute returns the undecorated function. This + The :func:`~singledispatch.register` attribute returns the undecorated function. This enables decorator stacking, :mod:`pickling`, and the creation of unit tests for each variant independently:: @@ -650,10 +659,10 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 .. versionchanged:: 3.7 - The :func:`register` attribute now supports using type annotations. + The :func:`~singledispatch.register` attribute now supports using type annotations. .. versionchanged:: 3.11 - The :func:`register` attribute now supports + The :func:`~singledispatch.register` attribute now supports :class:`typing.Union` as a type annotation. @@ -783,7 +792,7 @@ The :mod:`functools` module defines the following functions: 'Docstring' Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` + would have been ``'wrapper'``, and the docstring of the original :func:`!example` would have been lost. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 7ccb0e6bdf9406..2ef5c4b35a25cc 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -60,7 +60,7 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. @@ -83,13 +83,13 @@ The :mod:`gc` module provides the following functions: returned. If *generation* is not ``None``, return only the objects as follows: * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 1: No objects, as there is no generation 1 (as of Python 3.14) * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Generation 1 is removed .. audit-event:: gc.get_objects generation gc.get_objects @@ -142,7 +142,7 @@ The :mod:`gc` module provides the following functions: See `Garbage collector design `_ for more information. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 *threshold2* is ignored diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c9d96085ef739d..c59014a6f5bdce 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -26,7 +26,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -59,6 +59,11 @@ The module defines the following items: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: next + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -67,7 +72,7 @@ The module defines the following items: .. versionadded:: 3.8 -.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) +.. class:: GzipFile(filename=None, mode=None, compresslevel=6, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the methods of a :term:`file object`, with the exception of the :meth:`~io.IOBase.truncate` @@ -181,8 +186,13 @@ The module defines the following items: Remove the ``filename`` attribute, use the :attr:`~GzipFile.name` attribute instead. + .. versionchanged:: next + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. + -.. function:: compress(data, compresslevel=9, *, mtime=0) +.. function:: compress(data, compresslevel=6, *, mtime=0) Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in @@ -206,6 +216,10 @@ The module defines the following items: The *mtime* parameter now defaults to 0 for reproducible output. For the previous behaviour of using the current time, pass ``None`` to *mtime*. + .. versionchanged:: next + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. .. function:: decompress(data) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d6692033b2d4c3..57076c38086c79 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -12,6 +12,9 @@ -------------- This module implements the HMAC algorithm as described by :rfc:`2104`. +The interface allows to use any hash function with a *fixed* digest size. +In particular, extendable output functions such as SHAKE-128 or SHAKE-256 +cannot be used with HMAC. .. function:: new(key, msg=None, digestmod) diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 2835c8d0eb711e..07f5ebf57c9b54 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -34,7 +34,7 @@ The module provides the following classes: .. class:: HTTPConnection(host, port=None[, timeout], source_address=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) An :class:`HTTPConnection` instance represents one transaction with an HTTP server. It should be instantiated by passing it a host and optional port @@ -46,7 +46,9 @@ The module provides the following classes: The optional *source_address* parameter may be a tuple of a (host, port) to use as the source address the HTTP connection is made from. The optional *blocksize* parameter sets the buffer size in bytes for - sending a file-like message body. + sending a file-like message body. The optional *max_response_headers* + parameter sets the maximum number of allowed response headers to help + prevent denial-of-service attacks, otherwise the default value (100) is used. For example, the following calls all create instances that connect to the server at the same host and port:: @@ -66,10 +68,13 @@ The module provides the following classes: .. versionchanged:: 3.7 *blocksize* parameter was added. + .. versionchanged:: next + *max_response_headers* parameter was added. + .. class:: HTTPSConnection(host, port=None, *[, timeout], \ source_address=None, context=None, \ - blocksize=8192) + blocksize=8192, max_response_headers=None) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it @@ -109,6 +114,9 @@ The module provides the following classes: The deprecated *key_file*, *cert_file* and *check_hostname* parameters have been removed. + .. versionchanged:: next + *max_response_headers* parameter was added. + .. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) @@ -416,6 +424,14 @@ HTTPConnection Objects .. versionadded:: 3.7 +.. attribute:: HTTPConnection.max_response_headers + + The maximum number of allowed response headers to help prevent denial-of-service + attacks. By default, the maximum number of allowed headers is set to 100. + + .. versionadded:: next + + As an alternative to using the :meth:`~HTTPConnection.request` method described above, you can also send your request step by step, by using the four functions below. diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 23ddecf873876d..251aea891c3f98 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -570,7 +570,7 @@ Netscape protocol strictness switches: Don't allow setting cookies whose path doesn't path-match request URI. -:attr:`strict_ns_domain` is a collection of flags. Its value is constructed by +:attr:`~DefaultCookiePolicy.strict_ns_domain` is a collection of flags. Its value is constructed by or-ing together (for example, ``DomainStrictNoDots|DomainStrictNonDomain`` means both flags are set). diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index e002198899c8b8..7a11f4fe069004 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -16,11 +16,12 @@ within *packages*. "Resources" are file-like resources associated with a module or package in Python. The resources may be contained directly in a package, within a subdirectory contained in that package, or adjacent to modules outside a -package. Resources may be text or binary. As a result, Python module sources -(.py) of a package and compilation artifacts (pycache) are technically -de-facto resources of that package. In practice, however, resources are -primarily those non-Python artifacts exposed specifically by the package -author. +package. Resources may be text or binary. As a result, a package's Python +module sources (.py), compilation artifacts (pycache), and installation +artifacts (like :func:`reserved filenames ` +in directories) are technically de-facto resources of that package. +In practice, however, resources are primarily those non-Python artifacts +exposed specifically by the package author. Resources can be opened or read in either binary or text mode. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index ea5a77028683b3..4f374be778d6b3 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -206,6 +206,10 @@ Functions :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks a :class:`~importlib.machinery.ModuleSpec`. + .. warning:: + This function is not thread-safe. Calling it from multiple threads can result + in unexpected behavior. It's recommended to use the :class:`threading.Lock` + or other synchronization primitives for thread-safe module reloading. :mod:`importlib.abc` -- Abstract base classes related to import --------------------------------------------------------------- diff --git a/Doc/library/io.rst b/Doc/library/io.rst index de5cab5aee649f..dfebccb5a9cb91 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -719,6 +719,9 @@ than raw I/O does. The optional argument *initial_bytes* is a :term:`bytes-like object` that contains initial data. + Methods may be used from multiple threads without external locking in + :term:`free threading` builds. + :class:`BytesIO` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index e5bdfbb144b65a..9e887d8e65741b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -240,7 +240,16 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_reserved - ``True`` if the address is otherwise IETF reserved. + ``True`` if the address is noted as reserved by the IETF. + For IPv4, this is only ``240.0.0.0/4``, the ``Reserved`` address block. + For IPv6, this is all addresses `allocated `__ as + ``Reserved by IETF`` for future use. + + .. note:: For IPv4, ``is_reserved`` is not related to the address block value of the + ``Reserved-by-Protocol`` column in iana-ipv4-special-registry_. + + .. caution:: For IPv6, ``fec0::/10`` a former Site-Local scoped address prefix is + currently excluded from that list (see :attr:`~IPv6Address.is_site_local` & :rfc:`3879`). .. attribute:: is_loopback @@ -261,6 +270,7 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. _iana-ipv6-address-space: https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml .. method:: IPv4Address.__format__(fmt) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 00925ae920aad9..aa46920d3526f0 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -47,7 +47,7 @@ Iterator Arguments Results Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) โ†’ 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) โ†’ ABC DEF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) โ†’ AB CD EF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') โ†’ A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) โ†’ A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) โ†’ A C E F`` @@ -181,7 +181,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) โ†’ ABC DEF G + # batched('ABCDEFG', 2) โ†’ AB CD EF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4509da5891685f..e37683e32d01f4 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1317,7 +1317,7 @@ functions. In Python versions earlier than 3.4, this function could also be passed a text level, and would return the corresponding numeric value of the level. This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. + Python 3.4, but reinstated in 3.4.2 in order to retain backward compatibility. .. function:: getHandlerByName(name) diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 03da0e4713c42d..55f2de07553d56 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -42,6 +42,8 @@ noted otherwise, all return values are floats. :func:`fabs(x) ` Absolute value of *x* :func:`floor(x) ` Floor of *x*, the largest integer less than or equal to *x* :func:`fma(x, y, z) ` Fused multiply-add operation: ``(x * y) + z`` +:func:`fmax(x, y) ` Maximum of two floating-point values +:func:`fmin(x, y) ` Minimum of two floating-point values :func:`fmod(x, y) ` Remainder of division ``x / y`` :func:`modf(x) ` Fractional and integer parts of *x* :func:`remainder(x, y) ` Remainder of *x* with respect to *y* @@ -248,6 +250,30 @@ Floating point arithmetic .. versionadded:: 3.13 +.. function:: fmax(x, y) + + Get the larger of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmax` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, ยงF.10.0.3 and ยงF.10.9.2). + + .. versionadded:: next + + +.. function:: fmin(x, y) + + Get the smaller of two floating-point values, treating NaNs as missing data. + + When both operands are (signed) NaNs or zeroes, return ``nan`` and ``0`` + respectively and the sign of the result is implementation-defined, that + is, :func:`!fmin` is not required to be sensitive to the sign of such + operands (see Annex F of the C11 standard, ยงF.10.0.3 and ยงF.10.9.3). + + .. versionadded:: next + + .. function:: fmod(x, y) Return the floating-point remainder of ``x / y``, @@ -804,7 +830,7 @@ Constants The mathematical constant *ฯ„* = 6.283185..., to available precision. Tau is a circle constant equal to 2\ *ฯ€*, the ratio of a circle's circumference to its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) - Wrong `_, and start celebrating + Wrong `_, and start celebrating `Tau day `_ by eating twice as much pie! .. versionadded:: 3.6 diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index fc3c1134f97c85..c80f78e614818e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -936,8 +936,13 @@ For an example of the usage of queues for interprocess communication see .. method:: close() - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`~Queue.get`, :meth:`~Queue.put` and :meth:`~Queue.empty` + methods must no longer be called. + + The background thread will quit once it has flushed all buffered data to the pipe. This is called automatically when the queue is garbage collected. @@ -1118,7 +1123,9 @@ Miscellaneous Return a context object which has the same attributes as the :mod:`multiprocessing` module. - If *method* is ``None`` then the default context is returned. + If *method* is ``None`` then the default context is returned. Note that if + the global start method has not been set, this will set it to the + default method. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1129,10 +1136,10 @@ Miscellaneous Return the name of start method used for starting processes. - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. + If the global start method has not been set and *allow_none* is + ``False``, then the start method is set to the default and the name + is returned. If the start method has not been set and *allow_none* is + ``True`` then ``None`` is returned. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index f72aee19d8f332..d27023cde4cfe9 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -64,7 +64,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: basename(path) +.. function:: basename(path, /) Return the base name of pathname *path*. This is the second element of the pair returned by passing *path* to the function :func:`split`. Note that @@ -94,7 +94,7 @@ the :mod:`glob` module.) Any iterable can now be passed, rather than just sequences. -.. function:: commonprefix(list) +.. function:: commonprefix(list, /) Return the longest path prefix (taken character-by-character) that is a prefix of all paths in *list*. If *list* is empty, return the empty string @@ -118,7 +118,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: dirname(path) +.. function:: dirname(path, /) Return the directory name of pathname *path*. This is the first element of the pair returned by passing *path* to the function :func:`split`. @@ -199,14 +199,14 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getatime(path) +.. function:: getatime(path, /) Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. -.. function:: getmtime(path) +.. function:: getmtime(path, /) Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). @@ -216,7 +216,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getctime(path) +.. function:: getctime(path, /) Return the system's ctime which, on some systems (like Unix) is the time of the last metadata change, and, on others (like Windows), is the creation time for *path*. @@ -228,7 +228,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getsize(path) +.. function:: getsize(path, /) Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -237,7 +237,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isabs(path) +.. function:: isabs(path, /) Return ``True`` if *path* is an absolute pathname. On Unix, that means it begins with a slash, on Windows that it begins with two (back)slashes, or a @@ -261,7 +261,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isdir(path) +.. function:: isdir(path, /) Return ``True`` if *path* is an :func:`existing ` directory. This follows symbolic links, so both :func:`islink` and :func:`isdir` can be true @@ -298,9 +298,10 @@ the :mod:`glob` module.) device than *path*, or whether :file:`{path}/..` and *path* point to the same i-node on the same device --- this should detect mount points for all Unix and POSIX variants. It is not able to reliably detect bind mounts on the - same filesystem. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. + same filesystem. On Linux systems, it will always return ``True`` for btrfs + subvolumes, even if they aren't mount points. On Windows, a drive letter root + and a share UNC are always mount points, and for any other path + ``GetVolumePathName`` is called to see if it is different from the input path. .. versionchanged:: 3.4 Added support for detecting non-root mount points on Windows. @@ -350,7 +351,7 @@ the :mod:`glob` module.) .. versionadded:: 3.13 -.. function:: join(path, *paths) +.. function:: join(path, /, *paths) Join one or more path segments intelligently. The return value is the concatenation of *path* and all members of *\*paths*, with exactly one @@ -371,7 +372,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object` for *path* and *paths*. -.. function:: normcase(path) +.. function:: normcase(path, /) Normalize the case of a pathname. On Windows, convert all characters in the pathname to lowercase, and also convert forward slashes to backward slashes. @@ -401,7 +402,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: realpath(path, *, strict=False) +.. function:: realpath(path, /, *, strict=False) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating @@ -470,7 +471,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samefile(path1, path2) +.. function:: samefile(path1, path2, /) Return ``True`` if both pathname arguments refer to the same file or directory. This is determined by the device number and i-node number and raises an @@ -497,7 +498,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samestat(stat1, stat2) +.. function:: samestat(stat1, stat2, /) Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. These structures may have been returned by :func:`os.fstat`, @@ -507,11 +508,8 @@ the :mod:`glob` module.) .. versionchanged:: 3.4 Added Windows support. - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: split(path) +.. function:: split(path, /) Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the last pathname component and *head* is everything leading up to that. The @@ -527,7 +525,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitdrive(path) +.. function:: splitdrive(path, /) Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either a mount point or the empty string. On systems which do not use drive @@ -552,7 +550,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitroot(path) +.. function:: splitroot(path, /) Split the pathname *path* into a 3-item tuple ``(drive, root, tail)`` where *drive* is a device name or mount point, *root* is a string of separators @@ -585,7 +583,7 @@ the :mod:`glob` module.) .. versionadded:: 3.12 -.. function:: splitext(path) +.. function:: splitext(path, /) Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == path``, and the extension, *ext*, is empty or begins with a period and contains at diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1e54cfec609bd2..45ec6c7a51b7b0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -108,6 +108,12 @@ Python UTF-8 Mode .. versionadded:: 3.7 See :pep:`540` for more details. +.. versionchanged:: next + + Python UTF-8 mode is now enabled by default (:pep:`686`). + It may be disabled with by setting :envvar:`PYTHONUTF8=0 ` as + an environment variable or by using the :option:`-X utf8=0 <-X>` command line option. + The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage of the UTF-8 encoding: @@ -139,31 +145,22 @@ level APIs also exhibit different default behaviours: default so that attempting to open a binary file in text mode is likely to raise an exception rather than producing nonsense data. -The :ref:`Python UTF-8 Mode ` is enabled if the LC_CTYPE locale is -``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read` -function). - -It can be enabled or disabled using the :option:`-X utf8 <-X>` command line -option and the :envvar:`PYTHONUTF8` environment variable. - -If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the -interpreter defaults to using the current locale settings, *unless* the current -locale is identified as a legacy ASCII-based locale (as described for -:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or -fails. In such legacy locales, the interpreter will default to enabling UTF-8 -mode unless explicitly instructed not to do so. - -The Python UTF-8 Mode can only be enabled at the Python startup. Its value +The :ref:`Python UTF-8 Mode ` is enabled by default. +It can be disabled using the :option:`-X utf8=0 <-X>` command line +option or the :envvar:`PYTHONUTF8=0 ` environment variable. +The Python UTF-8 Mode can only be disabled at Python startup. Its value can be read from :data:`sys.flags.utf8_mode `. +If the UTF-8 mode is disabled, the interpreter defaults to using +the current locale settings, *unless* the current locale is identified +as a legacy ASCII-based locale (as described for :envvar:`PYTHONCOERCECLOCALE`), +and locale coercion is either disabled or fails. +In such legacy locales, the interpreter will default to enabling UTF-8 mode +unless explicitly instructed not to do so. + See also the :ref:`UTF-8 mode on Windows ` and the :term:`filesystem encoding and error handler`. -.. seealso:: - - :pep:`686` - Python 3.15 will make :ref:`utf8-mode` default. - .. _os-procinfo: diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 47986a2d9602ee..ebf5756146df92 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -542,20 +542,6 @@ Pure paths provide the following methods and properties: Passing additional arguments is deprecated; if supplied, they are joined with *other*. -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - .. versionchanged:: 3.13 - Windows path names that contain a colon, or end with a dot or a space, - are considered reserved. UNC paths may be reserved. - - .. deprecated-removed:: 3.13 3.15 - This method is deprecated; use :func:`os.path.isreserved` to detect - reserved paths on Windows. - .. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 06de152a742f28..37df13f8a1eb8c 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -176,8 +176,8 @@ Cross platform :attr:`processor` is resolved late, on demand. Note: the first two attribute names differ from the names presented by - :func:`os.uname`, where they are named :attr:`sysname` and - :attr:`nodename`. + :func:`os.uname`, where they are named :attr:`!sysname` and + :attr:`!nodename`. Entries which cannot be determined are set to ``''``. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index b6e51dffc40157..c5b88c22325404 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -4,7 +4,7 @@ The Python Profilers ******************** -**Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` +**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py` -------------- @@ -14,23 +14,32 @@ Introduction to the profilers ============================= .. index:: + single: statistical profiling + single: profiling, statistical single: deterministic profiling single: profiling, deterministic -:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of +Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of Python programs. A :dfn:`profile` is a set of statistics that describes how often and for how long various parts of the program executed. These statistics can be formatted into reports via the :mod:`pstats` module. -The Python standard library provides two different implementations of the same -profiling interface: +The Python standard library provides three different profiling implementations: -1. :mod:`cProfile` is recommended for most users; it's a C extension with +**Statistical Profiler:** + +1. :mod:`profile.sample` provides statistical profiling of running Python processes + using periodic stack sampling. It can attach to any running Python process without + requiring code modification or restart, making it ideal for production debugging. + +**Deterministic Profilers:** + +2. :mod:`cProfile` is recommended for development and testing; it's a C extension with reasonable overhead that makes it suitable for profiling long-running programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted Czotter. -2. :mod:`profile`, a pure Python module whose interface is imitated by +3. :mod:`profile`, a pure Python module whose interface is imitated by :mod:`cProfile`, but which adds significant overhead to profiled programs. If you're trying to extend the profiler in some way, the task might be easier with this module. Originally designed and written by Jim Roskind. @@ -44,6 +53,77 @@ profiling interface: but not for C-level functions, and so the C code would seem faster than any Python one. +**Profiler Comparison:** + ++-------------------+----------------------+----------------------+----------------------+ +| Feature | Statistical | Deterministic | Deterministic | +| | (``profile.sample``) | (``cProfile``) | (``profile``) | ++===================+======================+======================+======================+ +| **Target** | Running process | Code you run | Code you run | ++-------------------+----------------------+----------------------+----------------------+ +| **Overhead** | Virtually none | Moderate | High | ++-------------------+----------------------+----------------------+----------------------+ +| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts | ++-------------------+----------------------+----------------------+----------------------+ +| **Setup** | Attach to any PID | Instrument code | Instrument code | ++-------------------+----------------------+----------------------+----------------------+ +| **Use Case** | Production debugging | Development/testing | Profiler extension | ++-------------------+----------------------+----------------------+----------------------+ +| **Implementation**| C extension | C extension | Pure Python | ++-------------------+----------------------+----------------------+----------------------+ + +.. note:: + + The statistical profiler (:mod:`profile.sample`) is recommended for most production + use cases due to its extremely low overhead and ability to profile running processes + without modification. It can attach to any Python process and collect performance + data with minimal impact on execution speed, making it ideal for debugging + performance issues in live applications. + + +.. _statistical-profiling: + +What Is Statistical Profiling? +============================== + +:dfn:`Statistical profiling` works by periodically interrupting a running +program to capture its current call stack. Rather than monitoring every +function entry and exit like deterministic profilers, it takes snapshots at +regular intervals to build a statistical picture of where the program spends +its time. + +The sampling profiler uses process memory reading (via system calls like +``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on +Windows) to attach to a running Python process and extract stack trace +information without requiring any code modification or restart of the target +process. This approach provides several key advantages over traditional +profiling methods. + +The fundamental principle is that if a function appears frequently in the +collected stack samples, it is likely consuming significant CPU time. By +analyzing thousands of samples, the profiler can accurately estimate the +relative time spent in different parts of the program. The statistical nature +means that while individual measurements may vary, the aggregate results +converge to represent the true performance characteristics of the application. + +Since statistical profiling operates externally to the target process, it +introduces virtually no overhead to the running program. The profiler process +runs separately and reads the target process memory without interrupting its +execution. This makes it suitable for profiling production systems where +performance impact must be minimized. + +The accuracy of statistical profiling improves with the number of samples +collected. Short-lived functions may be missed or underrepresented, while +long-running functions will be captured proportionally to their execution time. +This characteristic makes statistical profiling particularly effective for +identifying the most significant performance bottlenecks rather than providing +exhaustive coverage of all function calls. + +Statistical profiling excels at answering questions like "which functions +consume the most CPU time?" and "where should I focus optimization efforts?" +rather than "exactly how many times was this function called?" The trade-off +between precision and practicality makes it an invaluable tool for performance +analysis in real-world applications. .. _profile-instant: @@ -54,6 +134,18 @@ This section is provided for users that "don't want to read the manual." It provides a very brief overview, and allows a user to rapidly perform profiling on an existing application. +**Statistical Profiling (Recommended for Production):** + +To profile an existing running process:: + + python -m profile.sample 1234 + +To profile with custom settings:: + + python -m profile.sample -i 50 -d 30 1234 + +**Deterministic Profiling (Development/Testing):** + To profile a function that takes a single argument, you can do:: import cProfile @@ -121,8 +213,208 @@ results to a file by specifying a filename to the :func:`run` function:: The :class:`pstats.Stats` class reads profile results from a file and formats them in various ways. +.. _sampling-profiler-cli: + +Statistical Profiler Command Line Interface +=========================================== + +.. program:: profile.sample + +The :mod:`profile.sample` module can be invoked as a script to profile running processes:: + + python -m profile.sample [options] PID + +**Basic Usage Examples:** + +Profile process 1234 for 10 seconds with default settings:: + + python -m profile.sample 1234 + +Profile with custom interval and duration, save to file:: + + python -m profile.sample -i 50 -d 30 -o profile.stats 1234 + +Generate collapsed stacks to use with tools like `flamegraph.pl +`_:: + + python -m profile.sample --collapsed 1234 + +Profile all threads, sort by total time:: + + python -m profile.sample -a --sort-tottime 1234 + +Profile with real-time sampling statistics:: + + python -m profile.sample --realtime-stats 1234 + +**Command Line Options:** + +.. option:: PID + + Process ID of the Python process to profile (required) + +.. option:: -i, --interval INTERVAL + + Sampling interval in microseconds (default: 100) + +.. option:: -d, --duration DURATION + + Sampling duration in seconds (default: 10) + +.. option:: -a, --all-threads + + Sample all threads in the process instead of just the main thread + +.. option:: --realtime-stats + + Print real-time sampling statistics during profiling + +.. option:: --pstats + + Generate pstats output (default) + +.. option:: --collapsed + + Generate collapsed stack traces for flamegraphs + +.. option:: -o, --outfile OUTFILE + + Save output to a file + +**Sorting Options (pstats format only):** + +.. option:: --sort-nsamples + + Sort by number of direct samples + +.. option:: --sort-tottime + + Sort by total time + +.. option:: --sort-cumtime + + Sort by cumulative time (default) + +.. option:: --sort-sample-pct + + Sort by sample percentage + +.. option:: --sort-cumul-pct + + Sort by cumulative sample percentage + +.. option:: --sort-nsamples-cumul + + Sort by cumulative samples + +.. option:: --sort-name + + Sort by function name + +.. option:: -l, --limit LIMIT + + Limit the number of rows in the output (default: 15) + +.. option:: --no-summary + + Disable the summary section in the output + +**Understanding Statistical Profile Output:** + +The statistical profiler produces output similar to deterministic profilers but with different column meanings:: + + Profile Stats: + nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) + 45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data) + 23/23 6.4 15.230 6.4 15.230 :0(len) + +**Column Meanings:** + +- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack +- **sample%**: Percentage of total samples where function was directly executing +- **tottime**: Estimated time spent directly in this function +- **cumul%**: Percentage of samples where function was anywhere on call stack +- **cumtime**: Estimated cumulative time including called functions +- **filename:lineno(function)**: Location and name of the function + .. _profile-cli: +:mod:`profile.sample` Module Reference +======================================================= + +.. module:: profile.sample + :synopsis: Python statistical profiler. + +This section documents the programmatic interface for the :mod:`profile.sample` module. +For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information +about statistical profiling, see :ref:`statistical-profiling` + +.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) + + Sample a Python process and generate profiling data. + + This is the main entry point for statistical profiling. It creates a + :class:`SampleProfiler`, collects stack traces from the target process, and + outputs the results in the specified format. + + :param int pid: Process ID of the target Python process + :param int sort: Sort order for pstats output (default: 2 for cumulative time) + :param int sample_interval_usec: Sampling interval in microseconds (default: 100) + :param int duration_sec: Duration to sample in seconds (default: 10) + :param str filename: Output filename (None for stdout/default naming) + :param bool all_threads: Whether to sample all threads (default: False) + :param int limit: Maximum number of functions to display (default: None) + :param bool show_summary: Whether to show summary statistics (default: True) + :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') + :param bool realtime_stats: Whether to display real-time statistics (default: False) + + :raises ValueError: If output_format is not 'pstats' or 'collapsed' + + Examples:: + + # Basic usage - profile process 1234 for 10 seconds + import profile.sample + profile.sample.sample(1234) + + # Profile with custom settings + profile.sample.sample(1234, duration_sec=30, sample_interval_usec=50, all_threads=True) + + # Generate collapsed stack traces for flamegraph.pl + profile.sample.sample(1234, output_format='collapsed', filename='profile.collapsed') + +.. class:: SampleProfiler(pid, sample_interval_usec, all_threads) + + Low-level API for the statistical profiler. + + This profiler uses periodic stack sampling to collect performance data + from running Python processes with minimal overhead. It can attach to + any Python process by PID and collect stack traces at regular intervals. + + :param int pid: Process ID of the target Python process + :param int sample_interval_usec: Sampling interval in microseconds + :param bool all_threads: Whether to sample all threads or just the main thread + + .. method:: sample(collector, duration_sec=10) + + Sample the target process for the specified duration. + + Collects stack traces from the target process at regular intervals + and passes them to the provided collector for processing. + + :param collector: Object that implements ``collect()`` method to process stack traces + :param int duration_sec: Duration to sample in seconds (default: 10) + + The method tracks sampling statistics and can display real-time + information if realtime_stats is enabled. + +.. seealso:: + + :ref:`sampling-profiler-cli` + Command-line interface documentation for the statistical profiler. + +Deterministic Profiler Command Line Interface +============================================= + .. program:: cProfile The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to @@ -564,7 +856,7 @@ What Is Deterministic Profiling? call*, *function return*, and *exception* events are monitored, and precise timings are made for the intervals between these events (during which time the user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and +provided by the :mod:`profile.sample` module) periodically samples the effective instruction pointer, and deduces where time is being spent. The latter technique traditionally involves less overhead (as the code does not need to be instrumented), but provides only relative indications of where time is being spent. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2d57cff10a9278..5506ac828e5abe 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -16,11 +16,10 @@ references to these attributes should be marked using the :member: role. -.. warning:: +.. note:: - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. index:: single: Expat diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index fbbebcf4ed8f92..1b75582f0cf45b 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -187,9 +187,6 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item - in the queue. - Raises a :exc:`ValueError` if called more times than there were items placed in the queue. @@ -204,6 +201,9 @@ fully processed by daemon consumer threads. count of unfinished tasks drops to zero, :meth:`join` unblocks. +Waiting for task completion +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Example of how to wait for enqueued tasks to be completed:: import threading @@ -233,22 +233,39 @@ Example of how to wait for enqueued tasks to be completed:: Terminating queues ^^^^^^^^^^^^^^^^^^ -:class:`Queue` objects can be made to prevent further interaction by shutting -them down. +When no longer needed, :class:`Queue` objects can be wound down +until empty or terminated immediately with a hard shutdown. .. method:: Queue.shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise - :exc:`ShutDown`. + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`ShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`ShutDown` in the formerly blocked thread. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will + raise :exc:`ShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only raise once the - queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise - immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`ShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be - unblocked. If *immediate* is true, a task will be marked as done for each - remaining item in the queue, which may unblock callers of - :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index ef0cfb0e76cef6..b1120b3a4d8eb4 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -447,6 +447,11 @@ Alternative Generator Override this method in subclasses to customise the :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + .. method:: Random.randbytes(n) + + Override this method in subclasses to customise the + :meth:`~random.randbytes` behaviour of :class:`!Random` instances. + .. class:: SystemRandom([seed]) diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst index a573c98f73eb0a..70c359cc1c0fc3 100644 --- a/Doc/library/security_warnings.rst +++ b/Doc/library/security_warnings.rst @@ -28,7 +28,7 @@ The following modules have specific security considerations: ` * :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race conditions ` -* :mod:`xml`: :ref:`XML vulnerabilities ` +* :mod:`xml`: :ref:`XML security ` * :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume exhaustion ` diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 23a2e0c3d0c758..b88fe4157bdc29 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -17,7 +17,8 @@ This includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are ordinary strings. -.. function:: open(filename, flag='c', protocol=None, writeback=False) +.. function:: open(filename, flag='c', protocol=None, writeback=False, *, \ + serializer=None, deserializer=None) Open a persistent dictionary. The filename specified is the base filename for the underlying database. As a side-effect, an extension may be added to the @@ -41,6 +42,21 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + for serializing and deserializing. This can be changed by supplying + *serializer* and *deserializer*, respectively. + + The *serializer* argument must be a callable which takes an object ``obj`` + and the *protocol* as inputs and returns the representation ``obj`` as a + :term:`bytes-like object`; the *protocol* value may be ignored by the + serializer. + + The *deserializer* argument must be callable which takes a serialized object + given as a :class:`bytes` object and returns the corresponding object. + + A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* + is not, or vice-versa. + .. versionchanged:: 3.10 :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. @@ -48,6 +64,10 @@ lots of shared sub-objects. The keys are ordinary strings. .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. + .. versionchanged:: next + Accepts custom *serializer* and *deserializer* functions in place of + :func:`pickle.dumps` and :func:`pickle.loads`. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -124,12 +144,13 @@ Restrictions which can cause hard crashes when trying to read from the database. * :meth:`Shelf.reorganize` may not be available for all database packages and - may temporarely increase resource usage (especially disk space) when called. + may temporarily increase resource usage (especially disk space) when called. Additionally, it will never run automatically and instead needs to be called explicitly. -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') +.. class:: Shelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, serializer=None, deserializer=None) A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. @@ -147,6 +168,9 @@ Restrictions The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. + The *serializer* and *deserializer* parameters have the same interpretation + as in :func:`~shelve.open`. + A :class:`Shelf` object can also be used as a context manager, in which case it will be automatically closed when the :keyword:`with` block ends. @@ -161,8 +185,13 @@ Restrictions :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') + +.. class:: BsdDbShelf(dict, protocol=None, writeback=False, \ + keyencoding='utf-8', *, \ + serializer=None, deserializer=None) A subclass of :class:`Shelf` which exposes :meth:`!first`, :meth:`!next`, :meth:`!previous`, :meth:`!last` and :meth:`!set_location` methods. @@ -172,18 +201,27 @@ Restrictions modules. The *dict* object passed to the constructor must support those methods. This is generally accomplished by calling one of :func:`!bsddb.hashopen`, :func:`!bsddb.btopen` or :func:`!bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *protocol*, *writeback*, *keyencoding*, *serializer* and *deserializer* + parameters have the same interpretation as in :func:`~shelve.open`. + + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) +.. class:: DbfilenameShelf(filename, flag='c', protocol=None, \ + writeback=False, *, serializer=None, \ + deserializer=None) A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like object. The underlying file will be opened using :func:`dbm.open`. By default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. + optional *flag* parameter has the same interpretation as for the + :func:`.open` function. The optional *protocol*, *writeback*, *serializer* + and *deserializer* parameters have the same interpretation as in + :func:`~shelve.open`. + + .. versionchanged:: next + Added the *serializer* and *deserializer* parameters. .. _shelve-example: @@ -225,6 +263,20 @@ object):: d.close() # close it +Exceptions +---------- + +.. exception:: ShelveError + + Exception raised when one of the arguments *deserializer* and *serializer* + is missing in the :func:`~shelve.open`, :class:`Shelf`, :class:`BsdDbShelf` + and :class:`DbfilenameShelf`. + + The *deserializer* and *serializer* arguments must be given together. + + .. versionadded:: next + + .. seealso:: Module :mod:`dbm` diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2dde40c9d92f45..dde38498206c46 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -618,7 +618,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is - available), or "xztar" (if the :mod:`lzma` module is available). + available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" + (if the :mod:`compression.zstd` module is available). *root_dir* is a directory that will be the root directory of the archive, all paths in the archive will be relative to it; for example, @@ -673,6 +674,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own archiver for any existing formats, by using :func:`register_archive_format`. @@ -716,8 +719,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *extract_dir* is the name of the target directory where the archive is unpacked. If not provided, the current working directory is used. - *format* is the archive format: one of "zip", "tar", "gztar", "bztar", or - "xztar". Or any other format registered with + *format* is the archive format: one of "zip", "tar", "gztar", "bztar", + "xztar", or "zstdtar". Or any other format registered with :func:`register_unpack_format`. If not provided, :func:`unpack_archive` will use the archive file name extension and see if an unpacker was registered for that extension. In case none is found, @@ -789,6 +792,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own unpacker for any existing formats, by using :func:`register_unpack_format`. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 641e1f1de03a1d..a14af6d3d88df2 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2288,7 +2288,7 @@ This section shows recipes for common adapters and converters. def adapt_datetime_iso(val): """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" - return val.isoformat() + return val.replace(tzinfo=None).isoformat() def adapt_datetime_epoch(val): """Adapt datetime.datetime to Unix timestamp.""" diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index ae2e324d0abaa4..a9930183f9a400 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1078,8 +1078,9 @@ SSL Sockets (but passing a non-zero ``flags`` argument is not allowed) - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.sendfile` (it may be high-performant only when + the kernel TLS is enabled by setting :data:`~ssl.OP_ENABLE_KTLS` or when a + socket is plain-text, else :meth:`~socket.socket.send` will be used) - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop @@ -1113,6 +1114,11 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. + .. versionchanged:: next + Python now uses ``SSL_sendfile`` internally when possible. The + function sends a file more efficiently because it performs TLS encryption + in the kernel to avoid additional context switches. + SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.read(len=1024, buffer=None) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 394c302fd354b9..90683c0b00d78a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1223,7 +1223,9 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | | is replaced by the contents of | | | | the iterable *t* | | +------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | +| ``del s[i:j]`` | removes the elements of | | +| | ``s[i:j]`` from the list | | +| | (same as ``s[i:j] = []``) | | +------------------------------+--------------------------------+---------------------+ | ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | | | are replaced by those of *t* | | @@ -2673,9 +2675,9 @@ For example: lead to a number of common errors (such as failing to display tuples and dictionaries correctly). Using the newer :ref:`formatted string literals `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. + ($-strings) ` may help avoid these errors. + Each of these alternatives provides their own trade-offs and benefits of + simplicity, flexibility, and/or extensibility. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 23e15780075435..83e8ee2722ed8a 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -198,8 +198,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. +related to that of :ref:`formatted string literals ` and +:ref:`template string literals `, but it is less sophisticated +and, in particular, does not support arbitrary expressions. .. index:: single: {} (curly brackets); in string formatting @@ -264,6 +265,8 @@ Some simple format string examples:: "Weight in tons {0.weight}" # 'weight' attribute of first positional arg "Units destroyed: {players[0]}" # First element of keyword argument 'players'. +.. _formatstrings-conversion: + The *conversion* field causes a type coercion before formatting. Normally, the job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted @@ -306,7 +309,7 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). +:ref:`formatstrings`, :ref:`f-strings`, and :ref:`t-strings`). They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -789,10 +792,20 @@ Nesting arguments and more complex examples:: -.. _template-strings: +.. _template-strings-pep292: -Template strings ----------------- +Template strings ($-strings) +---------------------------- + +.. note:: + + The feature described here was introduced in Python 2.4. It is unrelated + to, and should not be confused with, the newer + :ref:`template strings ` and + :ref:`t-string literal syntax ` introduced in Python 3.14. + T-string literals evaluate to instances of a different + :class:`~string.templatelib.Template` class, found in the + :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst new file mode 100644 index 00000000000000..31b90d75f411f0 --- /dev/null +++ b/Doc/library/string.templatelib.rst @@ -0,0 +1,313 @@ +:mod:`!string.templatelib` --- Support for template string literals +=================================================================== + +.. module:: string.templatelib + :synopsis: Support for template string literals. + +**Source code:** :source:`Lib/string/templatelib.py` + +-------------- + +.. seealso:: + + * :ref:`Format strings ` + * :ref:`T-string literal syntax ` + + +.. _template-strings: + +Template strings +---------------- + +.. versionadded:: 3.14 + +Template strings are a formatting mechanism that allows for deep control over +how strings are processed. You can create templates using +:ref:`t-string literal syntax `, which is identical to +:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. +While f-strings evaluate to ``str``, t-strings create a :class:`Template` +instance that gives you access to the static and interpolated (in curly braces) +parts of a string *before* they are combined. + + +.. _templatelib-template: + +Template +-------- + +The :class:`!Template` class describes the contents of a template string. + +:class:`!Template` instances are immutable: their attributes cannot be +reassigned. + +.. class:: Template(*args) + + Create a new :class:`!Template` object. + + :param args: A mix of strings and :class:`Interpolation` instances in any order. + :type args: str | Interpolation + + The most common way to create a :class:`!Template` instance is to use the + :ref:`t-string literal syntax `. This syntax is identical to that of + :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + + >>> name = "World" + >>> template = t"Hello {name}!" + >>> type(template) + + + Templates ars stored as sequences of literal :attr:`~Template.strings` + and dynamic :attr:`~Template.interpolations`. + A :attr:`~Template.values` attribute holds the interpolation values: + + >>> template.strings + ('Hello ', '!') + >>> template.interpolations + (Interpolation('World', ...),) + >>> template.values + ('World',) + + The :attr:`!strings` tuple has one more element than :attr:`!interpolations` + and :attr:`!values`; the interpolations โ€œbelongโ€ between the strings. + This may be easier to understand when tuples are aligned:: + + template.strings: ('Hello ', '!') + template.values: ( 'World', ) + + While literal syntax is the most common way to create :class:`!Template` + instances, it is also possible to create them directly using the constructor: + + >>> from string.templatelib import Interpolation, Template + >>> name = "World" + >>> template = Template("Hello, ", Interpolation(name, "name"), "!") + >>> list(template) + ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + + If two or more consecutive strings are passed, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: + + >>> from string.templatelib import Template + >>> template = Template("Hello ", "World", "!") + >>> template.strings + ('Hello World!',) + + If two or more consecutive interpolations are passed, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: + + >>> from string.templatelib import Interpolation, Template + >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) + >>> template.strings + ('', '', '') + + .. attribute:: strings + :type: tuple[str, ...] + + A :ref:`tuple ` of the static strings in the template. + + >>> name = "World" + >>> t"Hello {name}!".strings + ('Hello ', '!') + + Empty strings *are* included in the tuple: + + >>> name = "World" + >>> t"Hello {name}{name}!".strings + ('Hello ', '', '!') + + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: + + >>> t"".strings + ('',) + >>> t"".values + () + >>> t"{'cheese'}".strings + ('', '') + >>> t"{'cheese'}".values + ('cheese',) + + .. attribute:: interpolations + :type: tuple[Interpolation, ...] + + A tuple of the interpolations in the template. + + >>> name = "World" + >>> t"Hello {name}!".interpolations + (Interpolation('World', 'name', None, ''),) + + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: + + >>> t"Hello!".interpolations + () + + .. attribute:: values + :type: tuple[Any, ...] + + A tuple of all interpolated values in the template. + + >>> name = "World" + >>> t"Hello {name}!".values + ('World',) + + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is equivalent to + ``tuple(i.value for i in template.interpolations)``. + + .. describe:: iter(template) + + Iterate over the template, yielding each string and + :class:`Interpolation` in order. + + >>> name = "World" + >>> list(t"Hello {name}!") + ['Hello ', Interpolation('World', 'name', None, ''), '!'] + + Empty strings are *not* included in the iteration: + + >>> name = "World" + >>> list(t"Hello {name}{name}") + ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + + .. describe:: template + other + template += other + + Concatenate this template with another, returning a new + :class:`!Template` instance: + + >>> name = "World" + >>> list(t"Hello " + t"there {name}!") + ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + + Concatenation between a :class:`!Template` and a ``str`` is *not* supported. + This is because it is ambiguous whether the string should be treated as + a static string or an interpolation. If you want to concatenate a + :class:`!Template` with a string, you should either wrap the string + directly in a :class:`!Template` (to treat it as a static string) or use + an :class:`!Interpolation` (to treat it as dynamic): + + >>> from string.templatelib import Template, Interpolation + >>> template = t"Hello " + >>> # Treat "there " as a static string + >>> template += Template("there ") + >>> # Treat name as an interpolation + >>> name = "World" + >>> template += Template(Interpolation(name, "name")) + >>> list(template) + ['Hello there ', Interpolation('World', 'name', None, '')] + + +.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + + Create a new :class:`!Interpolation` object. + + :param value: The evaluated, in-scope result of the interpolation. + :type value: object + + :param expression: The text of a valid Python expression, or an empty string. + :type expression: str + + :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. + :type conversion: ``Literal["a", "r", "s"] | None`` + + :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. + :type format_spec: str + + The :class:`!Interpolation` type represents an expression inside a template string. + + :class:`!Interpolation` instances are immutable: their attributes cannot be + reassigned. + + .. attribute:: value + + :returns: The evaluated value of the interpolation. + :type: object + + >>> t"{1 + 2}".interpolations[0].value + 3 + + .. attribute:: expression + + :returns: The text of a valid Python expression, or an empty string. + :type: str + + The :attr:`~Interpolation.expression` is the original text of the + interpolation's Python expression, if the interpolation was created + from a t-string literal. Developers creating interpolations manually + should either set this to an empty string or choose a suitable valid + Python expression. + + >>> t"{1 + 2}".interpolations[0].expression + '1 + 2' + + .. attribute:: conversion + + :returns: The conversion to apply to the value, or ``None``. + :type: ``Literal["a", "r", "s"] | None`` + + The :attr:`!Interpolation.conversion` is the optional conversion to apply + to the value: + + >>> t"{1 + 2!a}".interpolations[0].conversion + 'a' + + .. note:: + + Unlike f-strings, where conversions are applied automatically, + the expected behavior with t-strings is that code that *processes* the + :class:`!Template` will decide how to interpret and whether to apply + the :attr:`!Interpolation.conversion`. + + .. attribute:: format_spec + + :returns: The format specification to apply to the value. + :type: str + + The :attr:`!Interpolation.format_spec` is an optional, arbitrary string + used as the format specification to present the value: + + >>> t"{1 + 2:.2f}".interpolations[0].format_spec + '.2f' + + .. note:: + + Unlike f-strings, where format specifications are applied automatically + via the :func:`format` protocol, the expected behavior with + t-strings is that code that *processes* the :class:`!Template` will + decide how to interpret and whether to apply the format specification. + As a result, :attr:`!Interpolation.format_spec` values in + :class:`!Template` instances can be arbitrary strings, even those that + do not necessarily conform to the rules of Python's :func:`format` + protocol. + + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec) + ... + 3.0 1 + 2 None .2f + + +Helper functions +---------------- + +.. function:: convert(obj, /, conversion) + + Applies formatted string literal :ref:`conversion ` + semantics to the given object *obj*. + This is frequently useful for custom template string processing logic. + + Three conversion flags are currently supported: + + * ``'s'`` which calls :func:`str` on the value, + * ``'r'`` which calls :func:`repr`, and + * ``'a'`` which calls :func:`ascii`. + + If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0674074b8c0df6..0f986aa580b3c9 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -137,7 +137,8 @@ The following events are supported: .. monitoring-event:: PY_UNWIND - Exit from a Python function during exception unwinding. + Exit from a Python function during exception unwinding. This includes exceptions raised directly within the + function and that are allowed to continue to propagate. .. monitoring-event:: PY_YIELD @@ -171,7 +172,7 @@ events, use the expression ``PY_RETURN | PY_START``. if get_events(DEBUGGER_ID) == NO_EVENTS: ... -Events are divided into three groups: + Setting this event deactivates all events. .. _monitoring-event-local: @@ -243,20 +244,23 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike +:monitoring-event:`RAISE`. -Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` -event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable -when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for -performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. +Note that the :monitoring-event:`STOP_ITERATION` event and the +:monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are +equivalent, and are treated as interchangeable when generating events. +Implementations will favor :monitoring-event:`STOP_ITERATION` for performance +reasons, but may generate a :monitoring-event:`RAISE` event with a +:exc:`StopIteration`. Turning events on and off ------------------------- In order to monitor an event, it must be turned on and a corresponding callback -must be registered. -Events can be turned on or off by setting the events either globally or -for a particular code object. +must be registered. Events can be turned on or off by setting the events either +globally and/or for a particular code object. An event will trigger only once, +even if it is turned on both globally and locally. Setting events globally @@ -292,10 +296,6 @@ in Python (see :ref:`c-api-monitoring`). Activates all the local events for *code* which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not in use. -Local events add to global events, but do not mask them. -In other words, all global events will trigger for a code object, -regardless of the local events. - Disabling events '''''''''''''''' @@ -325,8 +325,6 @@ except for a few breakpoints. Registering callback functions ------------------------------ -To register a callable for events call - .. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None Registers the callable *func* for the *event* with the given *tool_id* @@ -335,13 +333,17 @@ To register a callable for events call it is unregistered and returned. Otherwise :func:`register_callback` returns ``None``. + .. audit-event:: sys.monitoring.register_callback func sys.monitoring.register_callback Functions can be unregistered by calling ``sys.monitoring.register_callback(tool_id, event, None)``. Callback functions can be registered and unregistered at any time. -Registering or unregistering a callback function will generate a :func:`sys.audit` event. +Callbacks are called only once regardless if the event is turned on both +globally and locally. As such, if an event could be turned on for both global +and local events by your code then the callback needs to be written to handle +either trigger. Callback function arguments @@ -353,37 +355,46 @@ Callback function arguments that there are no arguments to the call. When an active event occurs, the registered callback function is called. +Callback functions returning an object other than :data:`DISABLE` will have no effect. Different events will provide the callback function with different arguments, as follows: * :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object * :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, retval: object) -> object -* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN` + (*arg0* can be :data:`MISSING` specifically):: - func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, callable: object, arg0: object) -> object + *code* represents the code object where the call is being made, while + *callable* is the object that is about to be called (and thus + triggered the event). If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. + For instance methods, *callable* will be the function object as found on the + class with *arg0* set to the instance (i.e. the ``self`` argument to the + method). + * :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, exception: BaseException) -> object * :monitoring-event:`LINE`:: - func(code: CodeType, line_number: int) -> DISABLE | Any + func(code: CodeType, line_number: int) -> object * :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: - func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, destination_offset: int) -> object Note that the *destination_offset* is where the code will next execute. * :monitoring-event:`INSTRUCTION`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 1626a89a0731bf..52f0af31c68726 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -953,6 +953,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only This function should be used for internal and specialized purposes only. It is not guaranteed to exist in all implementations of Python. + .. versionadded:: 3.12 + .. function:: getobjects(limit[, type]) @@ -2189,8 +2191,11 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: api_version - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. + The C API version, equivalent to the C macro :c:macro:`PYTHON_API_VERSION`. + Defined for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. .. data:: version_info diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 7cec108a5bd41d..acaec5b592bf6e 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -18,8 +18,8 @@ higher-level functions in :ref:`shutil `. Some facts and figures: -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. +* reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and + :mod:`lzma` compressed archives if the respective modules are available. * read/write support for the POSIX.1-1988 (ustar) format. @@ -47,6 +47,10 @@ Some facts and figures: or paths outside of the destination. Previously, the filter strategy was equivalent to :func:`fully_trusted `. +.. versionchanged:: 3.14 + + Added support for Zstandard compression using :mod:`compression.zstd`. + .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) Return a :class:`TarFile` object for the pathname *name*. For detailed @@ -59,8 +63,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | mode | action | +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | + | ``'r'`` or | Open for reading with transparent | + | ``'r:*'`` | compression (recommended). | +------------------+---------------------------------------------+ | ``'r:'`` | Open for reading exclusively without | | | compression. | @@ -71,6 +75,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'r:xz'`` | Open for reading with lzma compression. | +------------------+---------------------------------------------+ + | ``'r:zst'`` | Open for reading with Zstandard compression.| + +------------------+---------------------------------------------+ | ``'x'`` or | Create a tarfile exclusively without | | ``'x:'`` | compression. | | | Raise a :exc:`FileExistsError` exception | @@ -88,10 +94,15 @@ Some facts and figures: | | Raise a :exc:`FileExistsError` exception | | | if it already exists. | +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | + | ``'x:zst'`` | Create a tarfile with Zstandard compression.| + | | Raise a :exc:`FileExistsError` exception | + | | if it already exists. | +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | + | ``'a'`` or | Open for appending with no compression. The | + | ``'a:'`` | file is created if it does not exist. | + +------------------+---------------------------------------------+ + | ``'w'`` or | Open for uncompressed writing. | + | ``'w:'`` | | +------------------+---------------------------------------------+ | ``'w:gz'`` | Open for gzip compressed writing. | +------------------+---------------------------------------------+ @@ -99,6 +110,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'w:xz'`` | Open for lzma compressed writing. | +------------------+---------------------------------------------+ + | ``'w:zst'`` | Open for Zstandard compressed writing. | + +------------------+---------------------------------------------+ Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* is not suitable to open a certain (compressed) file for reading, @@ -110,11 +123,20 @@ Some facts and figures: For modes ``'w:gz'``, ``'x:gz'``, ``'w|gz'``, ``'w:bz2'``, ``'x:bz2'``, ``'w|bz2'``, :func:`tarfile.open` accepts the keyword argument - *compresslevel* (default ``9``) to specify the compression level of the file. + *compresslevel* (default ``6``) to specify the compression level of the file. For modes ``'w:xz'``, ``'x:xz'`` and ``'w|xz'``, :func:`tarfile.open` accepts the keyword argument *preset* to specify the compression level of the file. + For modes ``'w:zst'``, ``'x:zst'`` and ``'w|zst'``, :func:`tarfile.open` + accepts the keyword argument *level* to specify the compression level of + the file. The keyword argument *options* may also be passed, providing + advanced Zstandard compression parameters described by + :class:`~compression.zstd.CompressionParameter`. The keyword argument + *zstd_dict* can be passed to provide a :class:`~compression.zstd.ZstdDict`, + a Zstandard dictionary used to improve compression of smaller amounts of + data. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -146,6 +168,9 @@ Some facts and figures: | ``'r|xz'`` | Open an lzma compressed *stream* for | | | reading. | +-------------+--------------------------------------------+ + | ``'r|zst'`` | Open a Zstandard compressed *stream* for | + | | reading. | + +-------------+--------------------------------------------+ | ``'w|'`` | Open an uncompressed *stream* for writing. | +-------------+--------------------------------------------+ | ``'w|gz'`` | Open a gzip compressed *stream* for | @@ -157,6 +182,9 @@ Some facts and figures: | ``'w|xz'`` | Open an lzma compressed *stream* for | | | writing. | +-------------+--------------------------------------------+ + | ``'w|zst'`` | Open a Zstandard compressed *stream* for | + | | writing. | + +-------------+--------------------------------------------+ .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. @@ -170,6 +198,10 @@ Some facts and figures: .. versionchanged:: 3.14 The *preset* keyword argument also works for streams. + .. versionchanged:: next + The default compression level was reduced to 6 (down from 9). + It is the default level used by most compression tools and a better + tradeoff between speed and performance. .. class:: TarFile :noindex: @@ -1325,6 +1357,9 @@ Command-line options Examples -------- +Reading examples +~~~~~~~~~~~~~~~~~~~ + How to extract an entire tar archive to the current working directory:: import tarfile @@ -1347,6 +1382,23 @@ a generator function instead of a list:: tar.extractall(members=py_files(tar)) tar.close() +How to read a gzip compressed tar archive and display some member information:: + + import tarfile + tar = tarfile.open("sample.tar.gz", "r:gz") + for tarinfo in tar: + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") + if tarinfo.isreg(): + print("a regular file.") + elif tarinfo.isdir(): + print("a directory.") + else: + print("something else.") + tar.close() + +Writing examples +~~~~~~~~~~~~~~~~ + How to create an uncompressed tar archive from a list of filenames:: import tarfile @@ -1362,19 +1414,15 @@ The same example using the :keyword:`with` statement:: for name in ["foo", "bar", "quux"]: tar.add(name) -How to read a gzip compressed tar archive and display some member information:: +How to create and write an archive to stdout using +:data:`sys.stdout.buffer ` in the *fileobj* parameter +in :meth:`TarFile.add`:: - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() + import sys + import tarfile + with tarfile.open("sample.tar.gz", "w|gz", fileobj=sys.stdout.buffer) as tar: + for name in ["foo", "bar", "quux"]: + tar.add(name) How to create an archive and reset the user information using the *filter* parameter in :meth:`TarFile.add`:: diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0aae14c15a6104..9fdb21b8badbbf 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1384,6 +1384,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t .. versionadded:: 3.8 +.. function:: run_concurrently(worker_func, nthreads, args=(), kwargs={}) + + Run the worker function concurrently in multiple threads. + Re-raises an exception if any thread raises one, after all threads have + finished. + + :mod:`test.support.os_helper` --- Utilities for os tests ======================================================================== diff --git a/Doc/library/text.rst b/Doc/library/text.rst index 47b678434fc899..92e7dd9a53b80d 100644 --- a/Doc/library/text.rst +++ b/Doc/library/text.rst @@ -16,6 +16,7 @@ Python's built-in string type in :ref:`textseq`. .. toctree:: string.rst + string.templatelib.rst re.rst difflib.rst textwrap.rst diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 542493a82af94d..df9be68bf4f69d 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -306,10 +306,11 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.5 - The function is now always available and always system-wide. + The function is now always available and the clock is now the same for + all processes. .. versionchanged:: 3.10 - On macOS, the function is now system-wide. + On macOS, the clock is now the same for all processes. .. function:: monotonic_ns() -> int @@ -325,7 +326,8 @@ Functions Return the value (in fractional seconds) of a performance counter, i.e. a clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference + does include time elapsed during sleep. The clock is the same for all + processes. The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. @@ -340,7 +342,7 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.10 - On Windows, the function is now system-wide. + On Windows, the clock is now the same for all processes. .. versionchanged:: 3.13 Use the same clock as :func:`time.monotonic`. @@ -712,13 +714,18 @@ Functions Clock: - * On Windows, call ``GetSystemTimeAsFileTime()``. + * On Windows, call ``GetSystemTimePreciseAsFileTime()``. * Call ``clock_gettime(CLOCK_REALTIME)`` if available. * Otherwise, call ``gettimeofday()``. Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. +.. versionchanged:: 3.13 + + On Windows, calls ``GetSystemTimePreciseAsFileTime()`` instead of + ``GetSystemTimeAsFileTime()``. + .. function:: time_ns() -> int @@ -982,8 +989,8 @@ The following constant is the only parameter that can be sent to .. data:: CLOCK_REALTIME - System-wide real-time clock. Setting this clock requires appropriate - privileges. + Real-time clock. Setting this clock requires appropriate privileges. + The clock is the same for all processes. .. availability:: Unix. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 2bedd7fdd3c8c8..207024a7619902 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -333,6 +333,16 @@ Standard names are defined for the following types: :attr:`tb.tb_frame ` if ``tb`` is a traceback object. +.. data:: FrameLocalsProxyType + + The type of frame locals proxy objects, as found on the + :attr:`frame.f_locals` attribute. + + .. versionadded:: next + + .. seealso:: :pep:`667` + + .. data:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index dcdda1719bf593..ec96e8416120fa 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1131,7 +1131,7 @@ Test cases .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. - .. method:: assertLogs(logger=None, level=None) + .. method:: assertLogs(logger=None, level=None, formatter=None) A context manager to test that at least one message is logged on the *logger* or one of its children, with at least the given @@ -1146,6 +1146,10 @@ Test cases its string equivalent (for example either ``"ERROR"`` or :const:`logging.ERROR`). The default is :const:`logging.INFO`. + If given, *formatter* should be a :class:`logging.Formatter` object. + The default is a formatter with format string + ``"%(levelname)s:%(name)s:%(message)s"`` + The test passes if at least one message emitted inside the ``with`` block matches the *logger* and *level* conditions, otherwise it fails. @@ -1173,6 +1177,9 @@ Test cases .. versionadded:: 3.4 + .. versionchanged:: next + Now accepts a *formatter* to control how messages are formatted. + .. method:: assertNoLogs(logger=None, level=None) A context manager to test that no messages are logged on @@ -2556,7 +2563,7 @@ To add cleanup code that must be run even in the case of an exception, use .. versionadded:: 3.8 -.. classmethod:: enterModuleContext(cm) +.. function:: enterModuleContext(cm) Enter the supplied :term:`context manager`. If successful, also add its :meth:`~object.__exit__` method as a cleanup function by diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 58bd111b5cc374..e514b98fc5d553 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -210,6 +210,9 @@ The :mod:`urllib.request` module defines the following functions: Windows a UNC path is returned (as before), and on other platforms a :exc:`~urllib.error.URLError` is raised. + .. versionchanged:: 3.14 + The URL query and fragment components are discarded if present. + .. versionchanged:: 3.14 The *require_scheme* and *resolve_host* parameters were added. @@ -829,10 +832,13 @@ The following attribute and methods should only be used by classes derived from errors. It will be called automatically by the :class:`OpenerDirector` getting the error, and should not normally be called in other circumstances. - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. + :class:`OpenerDirector` will call this method with five positional arguments: + + 1. a :class:`Request` object, + #. a file-like object with the HTTP error body, + #. the three-digit code of the error, as a string, + #. the user-visible explanation of the code, as as string, and + #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of :func:`urlopen`. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index bed799aedfdfb1..de427fbafe71dc 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -22,10 +22,10 @@ The :mod:`!venv` module supports creating lightweight "virtual environments", each with their own independent set of Python packages installed in their :mod:`site` directories. A virtual environment is created on top of an existing -Python installation, known as the virtual environment's "base" Python, and may -optionally be isolated from the packages in the base environment, -so only those explicitly installed in the virtual environment are available. -See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +Python installation, known as the virtual environment's "base" Python, and by +default is isolated from the packages in the base environment, +so that only those explicitly installed in the virtual environment are +available. See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s :ref:`virtual environments documentation ` for more information. @@ -105,36 +105,52 @@ The command, if run with ``-h``, will show the available options:: Creates virtual Python environments in one or more target directories. - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory - if it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been - upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-files - Skips adding SCM ignore files to the environment - directory (Git is supported by default). - Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. +.. _venv-cli: +.. program:: venv + +.. option:: ENV_DIR + + A required argument specifying the directory to create the environment in. + +.. option:: --system-site-packages + + Give the virtual environment access to the system site-packages directory. + +.. option:: --symlinks + + Try to use symlinks rather than copies, when symlinks are not the default for the platform. + +.. option:: --copies + + Try to use copies rather than symlinks, even when symlinks are the default for the platform. + +.. option:: --clear + + Delete the contents of the environment directory if it already exists, before environment creation. + +.. option:: --upgrade + + Upgrade the environment directory to use this version of Python, assuming Python has been upgraded in-place. + +.. option:: --without-pip + + Skips installing or upgrading pip in the virtual environment (pip is bootstrapped by default). + +.. option:: --prompt + + Provides an alternative prompt prefix for this environment. + +.. option:: --upgrade-deps + + Upgrade core dependencies (pip) to the latest version in PyPI. + +.. option:: --without-scm-ignore-files + + Skips adding SCM ignore files to the environment directory (Git is supported by default). + .. versionchanged:: 3.4 Installs pip by default, added the ``--without-pip`` and ``--copies`` diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 00a18751207e7a..9ffedf7366a7b8 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -19,11 +19,10 @@ not already proficient with the DOM should consider using the :mod:`xml.etree.ElementTree` module for their XML processing instead. -.. warning:: +.. note:: - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. DOM applications typically start by parsing some XML into a DOM. With diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index fd96765cbe3c96..8bceeecd46393e 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -19,11 +19,10 @@ responsible for explicitly pulling events from the stream, looping over those events until either processing is finished or an error condition occurs. -.. warning:: +.. note:: - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 1daf6628013bf0..00075ac2a23e6b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -20,11 +20,10 @@ for parsing and creating XML data. The :mod:`!xml.etree.cElementTree` module is deprecated. -.. warning:: +.. note:: - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. Tutorial -------- diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index d495995398959d..28465219a1ac18 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -15,12 +15,10 @@ XML Processing Modules Python's interfaces for processing XML are grouped in the ``xml`` package. -.. warning:: +.. note:: - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defusedxml-package` sections. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -47,46 +45,22 @@ The XML handling submodules are: * :mod:`xml.parsers.expat`: the Expat parser binding +.. _xml-security: .. _xml-vulnerabilities: -XML vulnerabilities -------------------- +XML security +------------ -The XML processing modules are not secure against maliciously constructed data. An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or circumvent firewalls. -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. - -========================= ================== ================== ================== ================== ================== -kind sax etree minidom pulldom xmlrpc -========================= ================== ================== ================== ================== ================== -billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) -`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) -========================= ================== ================== ================== ================== ================== - -1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and - "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. -2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. -3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. -5. Since Python 3.7.1, external general entities are no longer processed by - default. -6. Expat 2.6.0 and newer is not vulnerable to denial of service - through quadratic runtime caused by parsing large tokens. - Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. +Expat versions lower that 2.6.0 may be vulnerable to "billion laughs", +"quadratic blowup" and "large tokens". Python may be vulnerable if it uses such +older versions of Expat as a system-provided library. +Check :const:`!pyexpat.EXPAT_VERSION`. + +:mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. billion laughs / exponential entity expansion @@ -103,16 +77,6 @@ quadratic blowup entity expansion efficient as the exponential case but it avoids triggering parser countermeasures that forbid deeply nested entities. -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -`DTD`_ retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - decompression bomb Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries that can parse compressed XML streams such as gzipped HTTP streams or @@ -126,21 +90,5 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. -The documentation for :pypi:`defusedxml` on PyPI has further information about -all known attack vectors with examples and references. - -.. _defusedxml-package: - -The :mod:`!defusedxml` Package ------------------------------- - -:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - - .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index c60e9e505f7544..5fa92645a440ce 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -18,11 +18,10 @@ SAX exceptions and the convenience functions which will be most used by users of the SAX API. -.. warning:: +.. note:: - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 654154cb43d6e5..547cb50be78a78 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -24,8 +24,8 @@ between conformable Python objects and XML on the wire. .. warning:: The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. versionchanged:: 3.5 diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 06169c7eca8b0c..2a8f6f8d5fc0de 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -20,8 +20,8 @@ servers written in Python. Servers can either be free standing, using .. warning:: The :mod:`xmlrpc.server` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. include:: ../includes/wasm-notavail.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6a4fa67332e179..a1261ec471c92e 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -129,14 +129,28 @@ The module defines the following items: .. versionadded:: 3.3 +.. data:: ZIP_ZSTANDARD + + The numeric constant for Zstandard compression. This requires the + :mod:`compression.zstd` module. + .. note:: - The ZIP file format specification has included support for bzip2 compression - since 2001, and for LZMA compression since 2006. However, some tools - (including older Python releases) do not support these compression - methods, and may either refuse to process the ZIP file altogether, - or fail to extract individual files. + In APPNOTE 6.3.7, the method ID ``20`` was assigned to Zstandard + compression. This was changed in APPNOTE 6.3.8 to method ID ``93`` to + avoid conflicts, with method ID ``20`` being deprecated. For + compatibility, the :mod:`!zipfile` module reads both method IDs but will + only write data with method ID ``93``. + + .. versionadded:: 3.14 + +.. note:: + The ZIP file format specification has included support for bzip2 compression + since 2001, for LZMA compression since 2006, and Zstandard compression since + 2020. However, some tools (including older Python releases) do not support + these compression methods, and may either refuse to process the ZIP file + altogether, or fail to extract individual files. .. seealso:: @@ -176,10 +190,11 @@ ZipFile Objects *compression* is the ZIP compression method to use when writing the archive, and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If - :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified - but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not + :const:`ZIP_BZIP2`, :const:`ZIP_LZMA`, or :const:`ZIP_ZSTANDARD`; + unrecognized values will cause :exc:`NotImplementedError` to be raised. If + :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2`, :const:`ZIP_LZMA`, or + :const:`ZIP_ZSTANDARD` is specified but the corresponding module + (:mod:`zlib`, :mod:`bz2`, :mod:`lzma`, or :mod:`compression.zstd`) is not available, :exc:`RuntimeError` is raised. The default is :const:`ZIP_STORED`. If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that @@ -194,6 +209,10 @@ ZipFile Objects (see :class:`zlib ` for more information). When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted (see :class:`bz2 ` for more information). + When using :const:`ZIP_ZSTANDARD` integers ``-131072`` through ``22`` are + commonly accepted (see + :attr:`CompressionParameter.compression_level ` + for more on retrieving valid values and their meaning). The *strict_timestamps* argument, when set to ``False``, allows to zip files older than 1980-01-01 at the cost of setting the @@ -415,9 +434,10 @@ ZipFile Objects read or append. *pwd* is the password used for encrypted files as a :class:`bytes` object and, if specified, overrides the default password set with :meth:`setpassword`. Calling :meth:`read` on a ZipFile that uses a compression method other than - :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or - :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also - be raised if the corresponding compression module is not available. + :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2`, + :const:`ZIP_LZMA`, or :const:`ZIP_ZSTANDARD` will raise a + :exc:`NotImplementedError`. An error will also be raised if the + corresponding compression module is not available. .. versionchanged:: 3.6 Calling :meth:`read` on a closed ZipFile will raise a :exc:`ValueError`. @@ -538,14 +558,6 @@ The following data attributes are also available: it should be no longer than 65535 bytes. Comments longer than this will be truncated. -.. attribute:: ZipFile.data_offset - - The offset to the start of ZIP data from the beginning of the file. When the - :class:`ZipFile` is opened in either mode ``'w'`` or ``'x'`` and the - underlying file does not support ``tell()``, the value will be ``None`` - instead. - - .. versionadded:: 3.14 .. _path-objects: diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e95fa3a6424e23..a416cbb4cc8eab 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -852,8 +852,8 @@ A literal pattern corresponds to most The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` are -not supported. +supported. Raw strings and byte strings are supported. :ref:`f-strings` +and :ref:`t-strings` are not supported. The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are for expressing :ref:`complex numbers `; they require a real number diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 24544a055c3ed2..9aca25e3214a16 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -133,13 +133,18 @@ Literals Python supports string and bytes literals and various numeric literals: -.. productionlist:: python-grammar - literal: `stringliteral` | `bytesliteral` | `NUMBER` +.. grammar-snippet:: + :group: python-grammar + + literal: `strings` | `NUMBER` Evaluation of a literal yields an object of the given type (string, bytes, integer, floating-point number, complex number) with the given value. The value may be approximated in the case of floating-point and imaginary (complex) -literals. See section :ref:`literals` for details. +literals. +See section :ref:`literals` for details. +See section :ref:`string-concatenation` for details on ``strings``. + .. index:: triple: immutable; data; type @@ -152,6 +157,58 @@ occurrence) may obtain the same object or a different object with the same value. +.. _string-concatenation: + +String literal concatenation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple adjacent string or bytes literals (delimited by whitespace), possibly +using different quoting conventions, are allowed, and their meaning is the same +as their concatenation:: + + >>> "hello" 'world' + "helloworld" + +Formally: + +.. grammar-snippet:: + :group: python-grammar + + strings: ( `STRING` | fstring)+ | tstring+ + +This feature is defined at the syntactical level, so it only works with literals. +To concatenate string expressions at run time, the '+' operator may be used:: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted strings, +and formatted string literals. +For example:: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes +needed, to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example:: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. +Also, template string literals may only be combined with other template +string literals:: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) + + .. _parenthesized: Parenthesized forms diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 55c148801d8559..1037feb691f6bc 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -10,11 +10,8 @@ error recovery. The notation used here is the same as in the preceding docs, and is described in the :ref:`notation ` section, -except for a few extra complications: +except for an extra complication: -* ``&e``: a positive lookahead (that is, ``e`` is required to match but - not consumed) -* ``!e``: a negative lookahead (that is, ``e`` is required *not* to match) * ``~`` ("cut"): commit to the current alternative and fail the rule even if this fails to parse diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 444acac374a690..c62240b18cfe55 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -145,15 +145,23 @@ The definition to the right of the colon uses the following syntax elements: * ``e?``: A question mark has exactly the same meaning as square brackets: the preceding item is optional. * ``(e)``: Parentheses are used for grouping. + +The following notation is only used in +:ref:`lexical definitions `. + * ``"a"..."z"``: Two literal characters separated by three dots mean a choice of any single character in the given (inclusive) range of ASCII characters. - This notation is only used in - :ref:`lexical definitions `. * ``<...>``: A phrase between angular brackets gives an informal description of the matched symbol (for example, ````), or an abbreviation that is defined in nearby text (for example, ````). - This notation is only used in - :ref:`lexical definitions `. + +.. _lexical-lookaheads: + +Some definitions also use *lookaheads*, which indicate that an element +must (or must not) match at a given position, but without consuming any input: + +* ``&e``: a positive lookahead (that is, ``e`` is required to match) +* ``!e``: a negative lookahead (that is, ``e`` is required *not* to match) The unary operators (``*``, ``+``, ``?``) bind as tightly as possible; the vertical bar (``|``) binds most loosely. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 567c70111c20ec..cf241829b71120 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -39,7 +39,8 @@ The end of a logical line is represented by the token :data:`~token.NEWLINE`. Statements cannot cross logical line boundaries except where :data:`!NEWLINE` is allowed by the syntax (e.g., between statements in compound statements). A logical line is constructed from one or more *physical lines* by following -the explicit or implicit *line joining* rules. +the :ref:`explicit ` or :ref:`implicit ` +*line joining* rules. .. _physical-lines: @@ -47,17 +48,30 @@ the explicit or implicit *line joining* rules. Physical lines -------------- -A physical line is a sequence of characters terminated by an end-of-line -sequence. In source files and strings, any of the standard platform line -termination sequences can be used - the Unix form using ASCII LF (linefeed), -the Windows form using the ASCII sequence CR LF (return followed by linefeed), -or the old Macintosh form using the ASCII CR (return) character. All of these -forms can be used equally, regardless of platform. The end of input also serves -as an implicit terminator for the final physical line. +A physical line is a sequence of characters terminated by one the following +end-of-line sequences: -When embedding Python, source code strings should be passed to Python APIs using -the standard C conventions for newline characters (the ``\n`` character, -representing ASCII LF, is the line terminator). +* the Unix form using ASCII LF (linefeed), +* the Windows form using the ASCII sequence CR LF (return followed by linefeed), +* the '`Classic Mac OS`__' form using the ASCII CR (return) character. + + __ https://en.wikipedia.org/wiki/Classic_Mac_OS + +Regardless of platform, each of these sequences is replaced by a single +ASCII LF (linefeed) character. +(This is done even inside :ref:`string literals `.) +Each line can use any of the sequences; they do not need to be consistent +within a file. + +The end of input also serves as an implicit terminator for the final +physical line. + +Formally: + +.. grammar-snippet:: + :group: python-grammar + + newline: | | .. _comments: @@ -106,6 +120,16 @@ If an encoding is declared, the encoding name must be recognized by Python encoding is used for all lexical analysis, including string literals, comments and identifiers. +All lexical analysis, including string literals, comments +and identifiers, works on Unicode text decoded using the source encoding. +Any Unicode code point, except the NUL control character, can appear in +Python source. + +.. grammar-snippet:: + :group: python-grammar + + source_character: + .. _explicit-joining: @@ -474,80 +498,110 @@ Literals Literals are notations for constant values of some built-in types. +In terms of lexical analysis, Python has :ref:`string, bytes ` +and :ref:`numeric ` literals. + +Other "literals" are lexically denoted using :ref:`keywords ` +(``None``, ``True``, ``False``) and the special +:ref:`ellipsis token ` (``...``). + .. index:: string literal, bytes literal, ASCII single: ' (single quote); string literal single: " (double quote); string literal - single: u'; string literal - single: u"; string literal .. _strings: String and Bytes literals -------------------------- +========================= -String literals are described by the following lexical definitions: +String literals are text enclosed in single quotes (``'``) or double +quotes (``"``). For example: -.. productionlist:: python-grammar - stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" | "t" | "T" - : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - : | "tr" | "Tr" | "tR" | "TR" | "rt" | "rT" | "Rt" | "RT" - shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' - longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' - shortstringitem: `shortstringchar` | `stringescapeseq` - longstringitem: `longstringchar` | `stringescapeseq` - shortstringchar: - longstringchar: - stringescapeseq: "\" +.. code-block:: python -.. productionlist:: python-grammar - bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' - longbytes: "'''" `longbytesitem`* "'''" | '"""' `longbytesitem`* '"""' - shortbytesitem: `shortbyteschar` | `bytesescapeseq` - longbytesitem: `longbyteschar` | `bytesescapeseq` - shortbyteschar: - longbyteschar: - bytesescapeseq: "\" - -One syntactic restriction not indicated by these productions is that whitespace -is not allowed between the :token:`~python-grammar:stringprefix` or -:token:`~python-grammar:bytesprefix` and the rest of the literal. The source -character set is defined by the encoding declaration; it is UTF-8 if no encoding -declaration is given in the source file; see section :ref:`encodings`. - -.. index:: triple-quoted string, Unicode Consortium, raw string + "spam" + 'eggs' + +The quote used to start the literal also terminates it, so a string literal +can only contain the other quote (except with escape sequences, see below). +For example: + +.. code-block:: python + + 'Say "Hello", please.' + "Don't do that!" + +Except for this limitation, the choice of quote character (``'`` or ``"``) +does not affect how the literal is parsed. + +Inside a string literal, the backslash (``\``) character introduces an +:dfn:`escape sequence`, which has special meaning depending on the character +after the backslash. +For example, ``\"`` denotes the double quote character, and does *not* end +the string: + +.. code-block:: pycon + + >>> print("Say \"Hello\" to everyone!") + Say "Hello" to everyone! + +See :ref:`escape sequences ` below for a full list of such +sequences, and more details. + + +.. index:: triple-quoted string single: """; string literal single: '''; string literal -In plain English: Both types of literals can be enclosed in matching single quotes -(``'``) or double quotes (``"``). They can also be enclosed in matching groups -of three single or double quotes (these are generally referred to as -*triple-quoted strings*). The backslash (``\``) character is used to give special -meaning to otherwise ordinary characters like ``n``, which means 'newline' when -escaped (``\n``). It can also be used to escape characters that otherwise have a -special meaning, such as newline, backslash itself, or the quote character. -See :ref:`escape sequences ` below for examples. +Triple-quoted strings +--------------------- -.. index:: - single: b'; bytes literal - single: b"; bytes literal +Strings can also be enclosed in matching groups of three single or double +quotes. +These are generally referred to as :dfn:`triple-quoted strings`:: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the literal, +if they are of the same kind (``'`` or ``"``) used at the start:: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained:: + + '''This triple-quoted string + continues on the next line.''' -Bytes literals are always prefixed with ``'b'`` or ``'B'``; they produce an -instance of the :class:`bytes` type instead of the :class:`str` type. They -may only contain ASCII characters; bytes with a numeric value of 128 or greater -must be expressed with escapes. .. index:: - single: r'; raw string literal - single: r"; raw string literal + single: u'; string literal + single: u"; string literal -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such constructs are called :dfn:`raw string literals` -and :dfn:`raw bytes literals` respectively and treat backslashes as -literal characters. As a result, in raw string literals, ``'\U'`` and ``'\u'`` -escapes are not treated specially. +String prefixes +--------------- + +String literals can have an optional :dfn:`prefix` that influences how the +content of the literal is parsed, for example: + +.. code-block:: python + + b"data" + f'{result=}' + +The allowed prefixes are: + +* ``b``: :ref:`Bytes literal ` +* ``r``: :ref:`Raw string ` +* ``f``: :ref:`Formatted string literal ` ("f-string") +* ``t``: :ref:`Template string literal ` ("t-string") +* ``u``: No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, ``B`` works the same as ``b``). +The ``r`` prefix can be combined with ``f``, ``t`` or ``b``, so ``fr``, +``rf``, ``tr``, ``rt``, ``br`` and ``rb`` are also valid prefixes. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym @@ -557,18 +611,35 @@ escapes are not treated specially. to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. -.. index:: - single: f'; formatted string literal - single: f"; formatted string literal -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw -formatted strings are possible, but formatted bytes literals are not. +Formal grammar +-------------- + +String literals, except :ref:`"f-strings" ` and +:ref:`"t-strings" `, are described by the +following lexical definitions. -In triple-quoted literals, unescaped newlines and quotes are allowed (and are -retained), except that three unescaped quotes in a row terminate the literal. (A -"quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) +These definitions use :ref:`negative lookaheads ` (``!``) +to indicate that an ending quote ends the literal. + +.. grammar-snippet:: + :group: python-grammar + + STRING: [`stringprefix`] (`stringcontent`) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "'" ( !"'" `stringitem`)* "'" + | '"' ( !'"' `stringitem`)* '"' + | "'''" ( !"'''" `longstringitem`)* "'''" + | '"""' ( !'"""' `longstringitem`)* '"""' + stringitem: `stringchar` | `stringescapeseq` + stringchar: + longstringitem: `stringitem` | newline + stringescapeseq: "\" + +Note that as in all lexical definitions, whitespace is significant. +In particular, the prefix (if any) must be immediately followed by the starting +quote. .. index:: physical line, escape sequence, Standard C, C single: \ (backslash); escape sequence @@ -587,120 +658,237 @@ retained), except that three unescaped quotes in a row terminate the literal. ( .. _escape-sequences: - Escape sequences -^^^^^^^^^^^^^^^^ +---------------- Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| ``\``\ | Backslash and newline ignored | \(1) | -+-------------------------+---------------------------------+-------+ -| ``\\`` | Backslash (``\``) | | -+-------------------------+---------------------------------+-------+ -| ``\'`` | Single quote (``'``) | | -+-------------------------+---------------------------------+-------+ -| ``\"`` | Double quote (``"``) | | -+-------------------------+---------------------------------+-------+ -| ``\a`` | ASCII Bell (BEL) | | -+-------------------------+---------------------------------+-------+ -| ``\b`` | ASCII Backspace (BS) | | -+-------------------------+---------------------------------+-------+ -| ``\f`` | ASCII Formfeed (FF) | | -+-------------------------+---------------------------------+-------+ -| ``\n`` | ASCII Linefeed (LF) | | -+-------------------------+---------------------------------+-------+ -| ``\r`` | ASCII Carriage Return (CR) | | -+-------------------------+---------------------------------+-------+ -| ``\t`` | ASCII Horizontal Tab (TAB) | | -+-------------------------+---------------------------------+-------+ -| ``\v`` | ASCII Vertical Tab (VT) | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\\\{ooo}` | Character with octal value | (2,4) | -| | *ooo* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\x{hh}` | Character with hex value *hh* | (3,4) | -+-------------------------+---------------------------------+-------+ - -Escape sequences only recognized in string literals are: - -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| :samp:`\\N\\{{name}\\}` | Character named *name* in the | \(5) | -| | Unicode database | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\u{xxxx}` | Character with 16-bit hex value | \(6) | -| | *xxxx* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\U{xxxxxxxx}` | Character with 32-bit hex value | \(7) | -| | *xxxxxxxx* | | -+-------------------------+---------------------------------+-------+ - -Notes: - -(1) - A backslash can be added at the end of a line to ignore the newline:: - - >>> 'This string will not include \ - ... backslashes or newline characters.' - 'This string will not include backslashes or newline characters.' - - The same result can be achieved using :ref:`triple-quoted strings `, - or parentheses and :ref:`string literal concatenation `. - - -(2) - As in Standard C, up to three octal digits are accepted. - - .. versionchanged:: 3.11 - Octal escapes with value larger than ``0o377`` produce a - :exc:`DeprecationWarning`. - - .. versionchanged:: 3.12 - Octal escapes with value larger than ``0o377`` produce a - :exc:`SyntaxWarning`. In a future Python version they will be eventually - a :exc:`SyntaxError`. - -(3) - Unlike in Standard C, exactly two hex digits are required. - -(4) - In a bytes literal, hexadecimal and octal escapes denote the byte with the - given value. In a string literal, these escapes denote a Unicode character - with the given value. - -(5) - .. versionchanged:: 3.3 - Support for name aliases [#]_ has been added. - -(6) - Exactly four hex digits are required. - -(7) - Any Unicode character can be encoded this way. Exactly eight hex digits - are required. +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Escape Sequence + * Meaning + * * ``\``\ + * :ref:`string-escape-ignore` + * * ``\\`` + * :ref:`Backslash ` + * * ``\'`` + * :ref:`Single quote ` + * * ``\"`` + * :ref:`Double quote ` + * * ``\a`` + * ASCII Bell (BEL) + * * ``\b`` + * ASCII Backspace (BS) + * * ``\f`` + * ASCII Formfeed (FF) + * * ``\n`` + * ASCII Linefeed (LF) + * * ``\r`` + * ASCII Carriage Return (CR) + * * ``\t`` + * ASCII Horizontal Tab (TAB) + * * ``\v`` + * ASCII Vertical Tab (VT) + * * :samp:`\\\\{ooo}` + * :ref:`string-escape-oct` + * * :samp:`\\x{hh}` + * :ref:`string-escape-hex` + * * :samp:`\\N\\{{name}\\}` + * :ref:`string-escape-named` + * * :samp:`\\u{xxxx}` + * :ref:`Hexadecimal Unicode character ` + * * :samp:`\\U{xxxxxxxx}` + * :ref:`Hexadecimal Unicode character ` + +.. _string-escape-ignore: + +Ignored end of line +^^^^^^^^^^^^^^^^^^^ + +A backslash can be added at the end of a line to ignore the newline:: + + >>> 'This string will not include \ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' + +The same result can be achieved using :ref:`triple-quoted strings `, +or parentheses and :ref:`string literal concatenation `. + +.. _string-escape-escaped-char: + +Escaped characters +^^^^^^^^^^^^^^^^^^ + +To include a backslash in a non-:ref:`raw ` Python string +literal, it must be doubled. The ``\\`` escape sequence denotes a single +backslash character:: + + >>> print('C:\\Program Files') + C:\Program Files + +Similarly, the ``\'`` and ``\"`` sequences denote the single and double +quote character, respectively:: + + >>> print('\' and \"') + ' and " + +.. _string-escape-oct: + +Octal character +^^^^^^^^^^^^^^^ + +The sequence :samp:`\\\\{ooo}` denotes a *character* with the octal (base 8) +value *ooo*:: + + >>> '\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given value. + +.. versionchanged:: 3.11 + Octal escapes with value larger than ``0o377`` (255) produce a + :exc:`DeprecationWarning`. + +.. versionchanged:: 3.12 + Octal escapes with value larger than ``0o377`` (255) produce a + :exc:`SyntaxWarning`. + In a future Python version they will raise a :exc:`SyntaxError`. + +.. _string-escape-hex: + +Hexadecimal character +^^^^^^^^^^^^^^^^^^^^^ + +The sequence :samp:`\\x{hh}` denotes a *character* with the hex (base 16) +value *hh*:: + + >>> '\x50' + 'P' + +Unlike in Standard C, exactly two hex digits are required. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given value. + +.. _string-escape-named: + +Named Unicode character +^^^^^^^^^^^^^^^^^^^^^^^ + +The sequence :samp:`\\N\\{{name}\\}` denotes a Unicode character +with the given *name*:: + + >>> '\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\N{SNAKE}' + '๐Ÿ' + +This sequence cannot appear in :ref:`bytes literals `. + +.. versionchanged:: 3.3 + Support for `name aliases `__ + has been added. + +.. _string-escape-long-hex: + +Hexadecimal Unicode characters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These sequences :samp:`\\u{xxxx}` and :samp:`\\U{xxxxxxxx}` denote the +Unicode character with the given hex (base 16) value. +Exactly four digits are required for ``\u``; exactly eight digits are +required for ``\U``. +The latter can encode any Unicode character. + +.. code-block:: pycon + + >>> '\u1234' + 'แˆด' + >>> '\U0001f40d' + '๐Ÿ' + +These sequences cannot appear in :ref:`bytes literals `. .. index:: unrecognized escape sequence -Unlike Standard C, all unrecognized escape sequences are left in the string -unchanged, i.e., *the backslash is left in the result*. (This behavior is -useful when debugging: if an escape sequence is mistyped, the resulting output -is more easily recognized as broken.) It is also important to note that the -escape sequences only recognized in string literals fall into the category of -unrecognized escapes for bytes literals. +Unrecognized escape sequences +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Unlike in Standard C, all unrecognized escape sequences are left in the string +unchanged, that is, *the backslash is left in the result*:: + + >>> print('\q') + \q + >>> list('\q') + ['\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in string +literals (``\N...``, ``\u...``, ``\U...``) fall into the category of +unrecognized escapes. .. versionchanged:: 3.6 Unrecognized escape sequences produce a :exc:`DeprecationWarning`. .. versionchanged:: 3.12 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In a future - Python version they will be eventually a :exc:`SyntaxError`. + Unrecognized escape sequences produce a :exc:`SyntaxWarning`. + In a future Python version they will raise a :exc:`SyntaxError`. + + +.. index:: + single: b'; bytes literal + single: b"; bytes literal + + +.. _bytes-literal: + +Bytes literals +-------------- + +:dfn:`Bytes literals` are always prefixed with ``'b'`` or ``'B'``; they produce an +instance of the :class:`bytes` type instead of the :class:`str` type. +They may only contain ASCII characters; bytes with a numeric value of 128 +or greater must be expressed with escape sequences (typically +:ref:`string-escape-hex` or :ref:`string-escape-oct`): + +.. code-block:: pycon + + >>> b'\x89PNG\r\n\x1a\n' + b'\x89PNG\r\n\x1a\n' + >>> list(b'\x89PNG\r\n\x1a\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence (typically +``\0`` or ``\x00``). + + +.. index:: + single: r'; raw string literal + single: r"; raw string literal + +.. _raw-strings: + +Raw string literals +------------------- + +Both string and bytes literals may optionally be prefixed with a letter ``'r'`` +or ``'R'``; such constructs are called :dfn:`raw string literals` +and :dfn:`raw bytes literals` respectively and treat backslashes as +literal characters. +As a result, in raw string literals, :ref:`escape sequences ` +are not treated specially: + +.. code-block:: pycon + + >>> r'\d{4}-\d{2}-\d{2}' + '\\d{4}-\\d{2}-\\d{2}' Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string @@ -712,29 +900,6 @@ that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. -.. _string-concatenation: - -String literal concatenation ----------------------------- - -Multiple adjacent string or bytes literals (delimited by whitespace), possibly -using different quoting conventions, are allowed, and their meaning is the same -as their concatenation. Thus, ``"hello" 'world'`` is equivalent to -``"helloworld"``. This feature can be used to reduce the number of backslashes -needed, to split long strings conveniently across long lines, or even to add -comments to parts of strings, for example:: - - re.compile("[A-Za-z_]" # letter or underscore - "[A-Za-z0-9_]*" # letter, digit or underscore - ) - -Note that this feature is defined at the syntactical level, but implemented at -compile time. The '+' operator must be used to concatenate string expressions -at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings), -and formatted string literals may be concatenated with plain string literals. - - .. index:: single: formatted string literal single: interpolated string literal @@ -742,6 +907,8 @@ and formatted string literals may be concatenated with plain string literals. single: string; interpolated literal single: f-string single: fstring + single: f'; formatted string literal + single: f"; formatted string literal single: {} (curly brackets); in formatted string literal single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal @@ -756,7 +923,7 @@ f-strings .. versionadded:: 3.6 A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain +that is prefixed with ``f`` or ``F``. These strings may contain replacement fields, which are expressions delimited by curly braces ``{}``. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. @@ -913,10 +1080,52 @@ See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. +.. _t-strings: +.. _template-string-literals: + +t-strings +--------- + +.. versionadded:: 3.14 + +A :dfn:`template string literal` or :dfn:`t-string` is a string literal +that is prefixed with ``t`` or ``T``. These strings follow the same +syntax and evaluation rules as :ref:`formatted string literals `, with +the following differences: + +- Rather than evaluating to a ``str`` object, t-strings evaluate to a + :class:`~string.templatelib.Template` object from the + :mod:`string.templatelib` module. + +- The :func:`format` protocol is not used. Instead, the format specifier and + conversions (if any) are passed to a new :class:`~string.templatelib.Interpolation` + object that is created for each evaluated expression. It is up to code that + processes the resulting :class:`~string.templatelib.Template` object to + decide how to handle format specifiers and conversions. + +- Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the expression ``{precision}`` before setting the ``format_spec`` + attribute of the resulting :class:`!Interpolation` object; if ``precision`` + is (for example) ``2``, the resulting format specifier will be ``'.2f'``. + +- When the equal sign ``'='`` is provided in an interpolation expression, the + resulting :class:`~string.templatelib.Template` object will have the expression + text along with a ``'='`` character placed in its + :attr:`~string.templatelib.Template.strings` attribute. The + :attr:`~string.templatelib.Template.interpolations` attribute will also + contain an ``Interpolation`` instance for the expression. By default, the + :attr:`~string.templatelib.Interpolation.conversion` attribute will be set to + ``'r'`` (that is, :func:`repr`), unless there is a conversion explicitly + specified (in which case it overrides the default) or a format specifier is + provided (in which case, the ``conversion`` defaults to ``None``). + + .. _numbers: Numeric literals ----------------- +================ .. index:: number, numeric literal, integer literal floating-point literal, hexadecimal literal @@ -949,7 +1158,7 @@ actually an expression composed of the unary operator '``-``' and the literal .. _integers: Integer literals -^^^^^^^^^^^^^^^^ +---------------- Integer literals denote whole numbers. For example:: @@ -1022,7 +1231,7 @@ Formally, integer literals are described by the following lexical definitions: .. _floating: Floating-point literals -^^^^^^^^^^^^^^^^^^^^^^^ +----------------------- Floating-point (float) literals, such as ``3.14`` or ``1.5``, denote :ref:`approximations of real numbers `. @@ -1084,7 +1293,7 @@ lexical definitions: .. _imaginary: Imaginary literals -^^^^^^^^^^^^^^^^^^ +------------------ Python has :ref:`complex number ` objects, but no complex literals. @@ -1172,14 +1381,26 @@ The following tokens serve as delimiters in the grammar: ( ) [ ] { } , : ! . ; @ = + +The period can also occur in floating-point and imaginary literals. + +.. _lexical-ellipsis: + +A sequence of three periods has a special meaning as an +:py:data:`Ellipsis` literal: + +.. code-block:: none + + ... + +The following *augmented assignment operators* serve +lexically as delimiters, but also perform an operation: + +.. code-block:: none + -> += -= *= /= //= %= @= &= |= ^= >>= <<= **= -The period can also occur in floating-point and imaginary literals. A sequence -of three periods has a special meaning as an ellipsis literal. The second half -of the list, the augmented assignment operators, serve lexically as delimiters, -but also perform an operation. - The following printing ASCII characters have special meaning as part of other tokens or are otherwise significant to the lexical analyzer: @@ -1194,7 +1415,3 @@ occurrence outside string literals and comments is an unconditional error: $ ? ` - -.. rubric:: Footnotes - -.. [#] https://www.unicode.org/Public/16.0.0/ucd/NameAliases.txt diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index e3bcb968128d96..31b2e567f11056 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -15,10 +15,7 @@ Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst -Doc/library/email.compat32-message.rst Doc/library/email.parser.rst -Doc/library/exceptions.rst -Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst @@ -30,7 +27,6 @@ Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickletools.rst -Doc/library/platform.rst Doc/library/profile.rst Doc/library/pyexpat.rst Doc/library/resource.rst @@ -73,6 +69,4 @@ Doc/whatsnew/3.5.rst Doc/whatsnew/3.6.rst Doc/whatsnew/3.7.rst Doc/whatsnew/3.8.rst -Doc/whatsnew/3.9.rst Doc/whatsnew/3.10.rst -Doc/whatsnew/3.11.rst diff --git a/Doc/tools/check-epub.py b/Doc/tools/check-epub.py new file mode 100644 index 00000000000000..693dc239c8ad58 --- /dev/null +++ b/Doc/tools/check-epub.py @@ -0,0 +1,24 @@ +import sys +from pathlib import Path + + +def main() -> int: + wrong_directory_msg = "Must run this script from the repo root" + if not Path("Doc").exists() or not Path("Doc").is_dir(): + raise RuntimeError(wrong_directory_msg) + + with Path("Doc/epubcheck.txt").open(encoding="UTF-8") as f: + messages = [message.split(" - ") for message in f.read().splitlines()] + + fatal_errors = [message for message in messages if message[0] == "FATAL"] + + if fatal_errors: + print("\nError: must not contain fatal errors:\n") + for error in fatal_errors: + print(" - ".join(error)) + + return len(fatal_errors) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 35b8c7cd8eb049..ea546c6a29df44 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,10 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. +The :mod:`string` module also contains support for so-called +:ref:`$-strings ` that offer yet another way to +substitute values into strings, using placeholders like ``$x`` and replacing +them with values from a dictionary. This syntax is easy to use, although +it offers much less control of the formatting. .. index:: single: formatted string literal diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 47bf7547b4ae1d..f8105cd5441fec 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -579,8 +579,8 @@ module for example, you might use:: from .. import formats from ..filters import equalizer -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use +Note that relative imports are based on the name of the current module's package. +Since the main module does not have a package, modules intended for use as the main module of a Python application must always use absolute imports. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index df81a330549147..2cda9587975ddc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -29,6 +29,9 @@ Features and minimum versions required to build CPython: * Tcl/Tk 8.5.12 for the :mod:`tkinter` module. +* `libmpdec `_ 2.5.0 + for the :mod:`decimal` module. + * Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. @@ -683,6 +686,13 @@ also be used to improve performance. not compiled. This includes both the functionality to schedule code to be executed and the functionality to receive code to be executed. + .. c:macro:: Py_REMOTE_DEBUG + + This macro is defined by default, unless Python is configured with + :option:`--without-remote-debug`. + + Note that even if the macro is defined, remote debugging may not be + available (for example, on an incompatible platform). .. versionadded:: 3.14 @@ -792,6 +802,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). + To improve ASan detection capabilities you may also want to combine this + with :option:`--without-pymalloc` to disable the specialized small-object + allocator whose allocations are not tracked by ASan. .. versionadded:: 3.6 diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 9628da3d2f6b12..c66308e61e89da 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -385,50 +385,77 @@ The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 - - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 + +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - ``PYTHON_MANAGER_DEFAULT`` + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` + - True to allow automatic installs when specifying a particular runtime + to launch. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -492,7 +519,7 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no @@ -638,46 +665,64 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 - - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 + +.. rubric:: Administrative configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Description + + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. .. _install-freethreaded-windows: @@ -716,70 +761,89 @@ issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 - - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " - "``pip`` gives me a ""command not found"" error when I type it in my - terminal.","Have you activated a virtual environment? Run the - ``.venv\Scripts\activate`` script in your terminal to activate. - " - "","The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, or alternatively - the ``python -m pip install --force pip`` command will recreate the - executables and show you the path to add to :envvar:`PATH`. These scripts are - separated for each runtime, and so you may need to add multiple paths. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 + +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your ``PYTHON_MANAGER_DEFAULT`` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, + or alternatively the ``python -m pip install --force pip`` command + will recreate the executables and show you the path to + add to :envvar:`PATH`. + These scripts are separated for each runtime, and so you may need to + add multiple paths. .. _windows-embeddable: @@ -1006,6 +1070,9 @@ UTF-8 mode ========== .. versionadded:: 3.7 +.. versionchanged:: next + + Python UTF-8 mode is now enabled by default (:pep:`686`). Windows still uses legacy encodings for the system encoding (the ANSI Code Page). Python uses it for the default encoding of text files (e.g. @@ -1014,20 +1081,22 @@ Page). Python uses it for the default encoding of text files (e.g. This may cause issues because UTF-8 is widely used on the internet and most Unix systems, including WSL (Windows Subsystem for Linux). -You can use the :ref:`Python UTF-8 Mode ` to change the default text -encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode ` via -the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment -variable. See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and -:ref:`setting-envvars` for how to modify environment variables. - -When the :ref:`Python UTF-8 Mode ` is enabled, you can still use the +The :ref:`Python UTF-8 Mode `, enabled by default, can help by +changing the default text encoding to UTF-8. +When the :ref:`UTF-8 mode ` is enabled, you can still use the system encoding (the ANSI Code Page) via the "mbcs" codec. -Note that adding ``PYTHONUTF8=1`` to the default environment variables -will affect all Python 3.7+ applications on your system. -If you have any Python 3.7+ applications which rely on the legacy -system encoding, it is recommended to set the environment variable -temporarily or use the ``-X utf8`` command line option. +You can disable the :ref:`Python UTF-8 Mode ` via +the ``-X utf8=0`` command line option, or the ``PYTHONUTF8=0`` environment +variable. See :envvar:`PYTHONUTF8` for disabling UTF-8 mode, and +:ref:`setting-envvars` for how to modify environment variables. + +.. hint:: + Adding ``PYTHONUTF8={0,1}`` to the default environment variables + will affect all Python 3.7+ applications on your system. + If you have any Python 3.7+ applications which rely on the legacy + system encoding, it is recommended to set the environment variable + temporarily or use the ``-X utf8`` command line option. .. note:: Even when UTF-8 mode is disabled, Python uses UTF-8 by default diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 2dd205dd2b8831..abf9677fd9cac5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1292,7 +1292,7 @@ This section covers specific optimizations independent of the (Contributed by Stefan Behnel in :gh:`68264`.) * Resizing lists is streamlined for the common case, - speeding up :meth:`list.append` by โ‰ˆ15% + speeding up :meth:`!list.append` by โ‰ˆ15% and simple :term:`list comprehension`\s by up to 20-30% (Contributed by Dennis Sweeney in :gh:`91165`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a65f59c0a72315..7cfdc287b7fad7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -2233,6 +2233,8 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-future.rst Removed diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ef7c36d8539dfd..126329b538db31 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1917,7 +1917,7 @@ New Deprecations * :mod:`pathlib`: - * Deprecate :meth:`.PurePath.is_reserved`, + * Deprecate :meth:`!.PurePath.is_reserved`, to be removed in Python 3.15. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Barney Gale in :gh:`88569`.) @@ -2546,6 +2546,8 @@ Deprecated C APIs .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst .. include:: ../deprecations/c-api-pending-removal-in-future.rst @@ -2592,7 +2594,7 @@ Build Changes * The :file:`configure` option :option:`--with-system-libmpdec` now defaults to ``yes``. - The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + The bundled copy of ``libmpdec`` will be removed in Python 3.16. * Python built with :file:`configure` :option:`--with-trace-refs` (tracing references) is now ABI compatible with the Python release build diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index a74d414ae4bb70..44ee2bbeb7761f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1058,6 +1058,30 @@ free-threaded build and false for the GIL-enabled build. (Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) +.. _whatsnew314-incremental-gc: + +Incremental garbage collection +------------------------------ + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + +There are now only two generations: young and old. +When :func:`gc.collect` is not called directly, the +GC is invoked a little less frequently. When invoked, it +collects the young generation and an increment of the +old generation, instead of collecting one or more generations. + +The behavior of :func:`!gc.collect` changes slightly: + +* ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. +* Other calls to :func:`!gc.collect` are unchanged. + +(Contributed by Mark Shannon in :gh:`108362`.) + + Other language changes ====================== @@ -1204,11 +1228,10 @@ argparse .. _whatsnew314-color-argparse: -* Introduced the optional *color* parameter to - :class:`argparse.ArgumentParser`, enabling color for help text. - This can be controlled by :ref:`environment variables - `. Color has also been enabled for help in the - :ref:`stdlib CLIs ` which use :mod:`!argparse`. +* Enable color for help text, which can be disabled with the optional *color* + parameter to :class:`argparse.ArgumentParser`. + This can also be controlled by :ref:`environment variables + `. (Contributed by Hugo van Kemenade in :gh:`130645`.) @@ -1486,6 +1509,36 @@ functools (Contributed by Sayandip Dutta in :gh:`125916`.) +gc +-- + +The cyclic garbage collector is now incremental, +which changes the meaning of the results of +:meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` +as well as :meth:`~gc.get_count` and :meth:`~gc.get_stats`. + +* For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is meaningless and is always zero. + +* :meth:`~gc.set_threshold` ignores any items after the second. + +* :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. + +In summary, code that attempted to manipulate the behavior of the cycle GC +may not work exactly as intended, but it is very unlikely to be harmful. +All other code will work just fine. + + getopt ------ @@ -2139,6 +2192,7 @@ urllib - Discard URL authority if it matches the local hostname. - Discard URL authority if it resolves to a local IP address when the new *resolve_host* argument is set to true. + - Discard URL query and fragment components. - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. @@ -2233,6 +2287,7 @@ asyncio (Contributed by Yury Selivanov, Pablo Galindo Salgado, and ลukasz Langa in :gh:`91048`.) + base64 ------ @@ -2241,6 +2296,15 @@ base64 (Contributed by Bรฉnรฉdikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) +gc +-- + +* The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + (Contributed by Mark Shannon in :gh:`108362`.) + + io --- * :mod:`io` which provides the built-in :func:`open` makes less system calls @@ -2260,8 +2324,7 @@ uuid * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster for 16-byte names and 20% faster for 1024-byte names. Performance for longer names remains unchanged. - * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster - respectively. + * :func:`~uuid.uuid4` is 30% faster. (Contributed by Bรฉnรฉdikt Tran in :gh:`128150`.) @@ -2640,7 +2703,9 @@ pty sqlite3 ------- -* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. +* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`; + use :data:`~sqlite3.sqlite_version` and :data:`~sqlite3.sqlite_version_info` + for the actual version number of the runtime SQLite library. (Contributed by Hugo van Kemenade in :gh:`118924`.) * Disallow using a sequence of parameters with named placeholders. @@ -2705,6 +2770,13 @@ Changes in the Python API Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + * The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` locale in some cases. This temporary change affects other threads. @@ -2868,11 +2940,6 @@ New features file. (Contributed by Victor Stinner in :gh:`127350`.) -* Add support of nullable arguments in :c:func:`PyArg_ParseTuple` and - similar functions. - Adding ``?`` after any format unit makes ``None`` be accepted as a value. - (Contributed by Serhiy Storchaka in :gh:`112068`.) - * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. @@ -2968,6 +3035,7 @@ Porting to Python 3.14 * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt` * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer` * ``_Py_fopen_obj()``: :c:func:`Py_fopen` + * ``PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` The `pythoncapi-compat project`_ can be used to get most of these new functions on Python 3.13 and older. @@ -3051,6 +3119,8 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst .. include:: ../deprecations/c-api-pending-removal-in-future.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index e58b4250f356c2..e8e2c1ed6047bf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -70,16 +70,137 @@ Summary --- release highlights New features ============ +.. _whatsnew315-sampling-profiler: + +High frequency statistical sampling profiler +-------------------------------------------- + +A new statistical sampling profiler has been added to the :mod:`profile` module as +:mod:`profile.sample`. This profiler enables low-overhead performance analysis of +running Python processes without requiring code modification or process restart. + +Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +every function call, the sampling profiler periodically captures stack traces from +running processes. This approach provides virtually zero overhead while achieving +sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler +available for Python (at the time of its contribution) and ideal for debugging +performance issues in production environments. + +Key features include: + +* **Zero-overhead profiling**: Attach to any running Python process without + affecting its performance +* **No code modification required**: Profile existing applications without restart +* **Real-time statistics**: Monitor sampling quality during data collection +* **Multiple output formats**: Generate both detailed statistics and flamegraph data +* **Thread-aware profiling**: Option to profile all threads or just the main thread + +Profile process 1234 for 10 seconds with default settings:: + + python -m profile.sample 1234 + +Profile with custom interval and duration, save to file:: + + python -m profile.sample -i 50 -d 30 -o profile.stats 1234 + +Generate collapsed stacks for flamegraph:: + + python -m profile.sample --collapsed 1234 + +Profile all threads and sort by total time:: + + python -m profile.sample -a --sort-tottime 1234 + +The profiler generates statistical estimates of where time is spent:: + + Real-time sampling stats: Mean: 100261.5Hz (9.97ยตs) Min: 86333.4Hz (11.58ยตs) Max: 118807.2Hz (8.42ยตs) Samples: 400001 + Captured 498841 samples in 5.00 seconds + Sample rate: 99768.04 samples/sec + Error rate: 0.72% + Profile Stats: + nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) + 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) + 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) + 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) + 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) + 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) + 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) + 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) + 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) + 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) + 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) + 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) + 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) + 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) + 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) + 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) + + Legend: + nsamples: Direct/Cumulative samples (direct executing / on call stack) + sample%: Percentage of total samples this function was directly executing + tottime: Estimated total time spent directly in this function + cumul%: Percentage of total samples when this function was on the call stack + cumtime: Estimated cumulative time (including time in called functions) + filename:lineno(function): Function location and name + + Summary of Interesting Functions: + + Functions with Highest Direct/Cumulative Ratio (Hot Spots): + 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) + 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) + 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) + + Functions with Highest Call Frequency (Indirect Calls): + 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) + 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) + 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) + + Functions with Highest Call Magnification (Cumulative/Direct): + 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) + 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) + 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) + +The profiler automatically identifies performance bottlenecks through statistical +analysis, highlighting functions with high CPU usage and call frequency patterns. + +This capability is particularly valuable for debugging performance issues in +production systems where traditional profiling approaches would be too intrusive. + +(Contributed by Pablo Galindo and Lรกszlรณ Kiss Kollรกr in :gh:`135953`.) Other language changes ====================== +* Python now uses UTF-8_ as the default encoding, independent of the system's + environment. This means that I/O operations without an explicit encoding, + e.g. ``open('flying-circus.txt')``, will use UTF-8. + UTF-8 is a widely-supported Unicode_ character encoding that has become a + *de facto* standard for representing text, including nearly every webpage + on the internet, many common file formats, programming languages, and more. + + This only applies when no ``encoding`` argument is given. For best + compatibility between versions of Python, ensure that an explicit ``encoding`` + argument is always provided. The :ref:`opt-in encoding warning ` + can be used to identify code that may be affected by this change. + The special special ``encoding='locale'`` argument uses the current locale + encoding, and has been supported since Python 3.10. + + To retain the previous behaviour, Python's UTF-8 mode may be disabled with + the :envvar:`PYTHONUTF8=0 ` environment variable or the + :option:`-X utf8=0 <-X>` command line option. + + .. seealso:: :pep:`686` for further details. + + .. _UTF-8: https://en.wikipedia.org/wiki/UTF-8 + .. _Unicode: https://home.unicode.org/ + + (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) + * Several error messages incorrectly using the term "argument" have been corrected. (Contributed by Stan Ulbrych in :gh:`133382`.) - New modules =========== @@ -109,13 +230,23 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +http.client +----------- + +* A new *max_response_headers* keyword-only parameter has been added to + :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` + constructors. This parameter overrides the default maximum number of allowed + response headers. + (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) + + math ---- * Add :func:`math.isnormal` and :func:`math.issubnormal` functions. (Contributed by Sergey B Kirpichev in :gh:`132908`.) -* Add :func:`math.signbit` function. +* Add :func:`math.fmax`, :func:`math.fmin` and :func:`math.signbit` functions. (Contributed by Bรฉnรฉdikt Tran in :gh:`135853`.) @@ -140,8 +271,15 @@ shelve sqlite3 ------- -* Support SQL keyword completion in the :mod:`sqlite3` command-line interface. - (Contributed by Long Tan in :gh:`133393`.) +* The :ref:`command-line interface ` has several new features: + + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) + + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and ลukasz Langa in :gh:`133461`) ssl @@ -174,6 +312,23 @@ tarfile and :cve:`2025-4435`.) +types +------ + +* Expose the write-through :func:`locals` proxy type + as :data:`types.FrameLocalsProxyType`. + This represents the type of the :attr:`frame.f_locals` attribute, + as described in :pep:`667`. + + +unittest +-------- + +* :func:`unittest.TestCase.assertLogs` will now accept a formatter + to control how messages are formatted. + (Contributed by Garry Cairns in :gh:`134567`.) + + zlib ---- @@ -237,6 +392,14 @@ http.server (Contributed by Bรฉnรฉdikt Tran in :gh:`133810`.) +pathlib +------- + +* Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + (Contributed by Nikita Sobolev in :gh:`133875`.) + + platform -------- @@ -245,6 +408,13 @@ platform (Contributed by Alexey Makridenko in :gh:`133604`.) +sre_* +----- + +* Removed :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + (Contributed by Stan Ulbrych in :gh:`135994`.) + + sysconfig --------- @@ -296,6 +466,11 @@ that may require changes to your code. Build changes ============= +* Removed implicit fallback to the bundled copy of the ``libmpdec`` library. + Now this should be explicitly enabled with :option:`--with-system-libmpdec` + set to ``no`` or with :option:`!--without-system-libmpdec`. + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + C API changes ============= @@ -327,10 +502,20 @@ Porting to Python 3.15 (Contributed by Serhiy Storchaka in :gh:`133595`.) +* Private functions promoted to public C APIs: + + The |pythoncapi_compat_project| can be used to get most of these new + functions on Python 3.14 and older. + + Deprecated C APIs ----------------- -* TODO +* For unsigned integer formats in :c:func:`PyArg_ParseTuple`, + accepting Python integers with value that is larger than the maximal value + for the C type or less than the minimal value for the corresponding + signed integer type of the same size is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`132629`.) .. Add C API deprecations above alphabetically, not here at the end. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 896e8f4a489649..40d4a27bff9fee 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -423,8 +423,8 @@ digests. It skips MD5 on platforms that block MD5 digest. fcntl ----- -Added constants :const:`~fcntl.F_OFD_GETLK`, :const:`~fcntl.F_OFD_SETLK` -and :const:`~fcntl.F_OFD_SETLKW`. +Added constants :const:`!fcntl.F_OFD_GETLK`, :const:`!fcntl.F_OFD_SETLK` +and :const:`!fcntl.F_OFD_SETLKW`. (Contributed by Donghee Na in :issue:`38602`.) ftplib @@ -644,7 +644,7 @@ attribute. random ------ -Added a new :attr:`random.Random.randbytes` method: generate random bytes. +Added a new :meth:`random.Random.randbytes` method: generate random bytes. (Contributed by Victor Stinner in :issue:`40286`.) signal @@ -776,7 +776,7 @@ Optimizations :pep:`590` vectorcall protocol. (Contributed by Donghee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) -* Optimized :func:`~set.difference_update` for the case when the other set +* Optimized :meth:`!set.difference_update` for the case when the other set is much larger than the base set. (Suggested by Evgeny Kapun with code contributed by Michele Orrรน in :issue:`8425`.) diff --git a/Grammar/python.gram b/Grammar/python.gram index a5ab769910b5ac..ff54e42111005a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -184,8 +184,9 @@ return_stmt[stmt_ty]: | 'return' a=[star_expressions] { _PyAST_Return(a, EXTRA) } raise_stmt[stmt_ty]: + | 'raise' a=expression 'from' b=expression { _PyAST_Raise(a, b, EXTRA) } | invalid_raise_stmt - | 'raise' a=expression b=['from' z=expression { z }] { _PyAST_Raise(a, b, EXTRA) } + | 'raise' a=expression { _PyAST_Raise(a, NULL, EXTRA) } | 'raise' { _PyAST_Raise(NULL, NULL, EXTRA) } pass_stmt[stmt_ty]: @@ -986,7 +987,10 @@ tstring[expr_ty] (memo): _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) } string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) } -strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } +strings[expr_ty] (memo): + | invalid_string_tstring_concat + | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } + | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } @@ -1552,6 +1556,12 @@ invalid_tstring_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") } +invalid_string_tstring_concat: + | a=(fstring|string)+ b[expr_ty]=tstring { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + | a=tstring+ b[expr_ty]=(fstring|string) { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a3..3f49b78947c9a6 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -45,19 +45,19 @@ # endif #endif -// gh-111506: The free-threaded build is not compatible with the limited API -// or the stable ABI. -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -# error "The limited API is not currently supported in the free-threaded build" -#endif +#if defined(Py_GIL_DISABLED) +# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) +# error "Py_LIMITED_API is not currently supported in the free-threaded build" +# endif -#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) -# include // __readgsqword() -#endif +# if defined(_MSC_VER) +# include // __readgsqword() +# endif -#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) -# include // __readgsqword() -#endif +# if defined(__MINGW32__) +# include // __readgsqword() +# endif +#endif // Py_GIL_DISABLED // Include Python header files #include "pyport.h" @@ -68,7 +68,7 @@ #include "pybuffer.h" #include "pystats.h" #include "pyatomic.h" -#include "lock.h" +#include "pylock.h" #include "critical_section.h" #include "object.h" #include "refcount.h" diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 35db3fb6a59ce6..4fc46fefb93a24 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -73,22 +73,32 @@ typedef struct PyCriticalSection2 PyCriticalSection2; PyAPI_FUNC(void) PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); +PyAPI_FUNC(void) +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + PyAPI_FUNC(void) PyCriticalSection_End(PyCriticalSection *c); PyAPI_FUNC(void) PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); +PyAPI_FUNC(void) +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + PyAPI_FUNC(void) PyCriticalSection2_End(PyCriticalSection2 *c); #ifndef Py_GIL_DISABLED # define Py_BEGIN_CRITICAL_SECTION(op) \ { +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { # define Py_END_CRITICAL_SECTION() \ } # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { # define Py_END_CRITICAL_SECTION2() \ } #else /* !Py_GIL_DISABLED */ @@ -118,6 +128,11 @@ struct PyCriticalSection2 { PyCriticalSection _py_cs; \ PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_BeginMutex(&_py_cs, mutex) + # define Py_END_CRITICAL_SECTION() \ PyCriticalSection_End(&_py_cs); \ } @@ -127,6 +142,11 @@ struct PyCriticalSection2 { PyCriticalSection2 _py_cs2; \ PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + # define Py_END_CRITICAL_SECTION2() \ PyCriticalSection2_End(&_py_cs2); \ } diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 7ce4acfeb7177d..1c979d91a40850 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -102,15 +102,14 @@ typedef struct PyPreConfig { /* Enable UTF-8 mode? (PEP 540) - Disabled by default (equals to 0). + If equal to 1, use the UTF-8 encoding and use "surrogateescape" for the + stdin & stdout error handlers. - Set to 1 by "-X utf8" and "-X utf8=1" command line options. - Set to 1 by PYTHONUTF8=1 environment variable. + Enabled by default (equal to 1; PEP 686), or if Py_UTF8Mode=1, + or if "-X utf8=1" or PYTHONUTF8=1. - Set to 0 by "-X utf8=0" and PYTHONUTF8=0. - - If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or - "POSIX", otherwise it is set to 0. Inherit Py_UTF8Mode value value. */ + Set to 0 by "-X utf8=0" or PYTHONUTF8=0. + */ int utf8_mode; /* If non-zero, enable the Python Development Mode. diff --git a/Include/cpython/lock.h b/Include/cpython/pylock.h similarity index 85% rename from Include/cpython/lock.h rename to Include/cpython/pylock.h index 8ee03e82f74dfd..63886fca28eae2 100644 --- a/Include/cpython/lock.h +++ b/Include/cpython/pylock.h @@ -36,6 +36,9 @@ PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); // exported function for unlocking the mutex PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); +// exported function for checking if the mutex is locked +PyAPI_FUNC(int) PyMutex_IsLocked(PyMutex *m); + // Locks the mutex. // // If the mutex is currently locked, the calling thread will be parked until @@ -61,3 +64,11 @@ _PyMutex_Unlock(PyMutex *m) } } #define PyMutex_Unlock _PyMutex_Unlock + +// Checks if the mutex is currently locked. +static inline int +_PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; +} +#define PyMutex_IsLocked _PyMutex_IsLocked diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 86c502730f478c..73e3bc44d6c9ca 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -109,7 +109,7 @@ struct _PyUnicodeObject_state { immediately follow the structure. utf8_length can be found in the length field; the utf8 pointer is equal to the data pointer. */ typedef struct { - /* There are 4 forms of Unicode strings: + /* There are 3 forms of Unicode strings: - compact ascii: diff --git a/Include/fileutils.h b/Include/fileutils.h index 1509198e45f0ca..00c37fa4602428 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -16,7 +16,7 @@ # define S_IFMT 0170000 #endif #ifndef S_IFLNK - // Windows doesn't define S_IFLNK, but posixmodule.c maps + // Windows doesn't define S_IFLNK, but fileutils.c maps // IO_REPARSE_TAG_SYMLINK to S_IFLNK. # define S_IFLNK 0120000 #endif diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index a04169f7fb84e8..19e93224174314 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -481,7 +481,7 @@ typedef struct mi_segment_s { struct mi_segment_s* next; // the list of freed segments in the cache (must be first field, see `segment.c:mi_segment_init`) size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) - size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long) size_t used; // count of pages in use uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` @@ -705,7 +705,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); // Thread Local data // ------------------------------------------------------ -// A "span" is is an available range of slices. The span queues keep +// A "span" is an available range of slices. The span queues keep // track of slice spans of at most the given `slice_count` (but more than the previous size class). typedef struct mi_span_queue_s { mi_slice_t* first; diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 62460c5f8fad30..2601de40737e85 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -21,16 +21,6 @@ extern "C" { #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ - { \ - PyCriticalSection _py_cs; \ - _PyCriticalSection_BeginMutex(&_py_cs, mutex) - -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ - { \ - PyCriticalSection2 _py_cs2; \ - _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - // Specialized version of critical section locking to safely use // PySequence_Fast APIs without the GIL. For performance, the argument *to* // PySequence_Fast() is provided to the macro, not the *result* of @@ -75,8 +65,6 @@ extern "C" { #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -119,6 +107,7 @@ _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) _PyCriticalSection_BeginSlow(c, m); } } +#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void _PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) @@ -194,6 +183,7 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) _PyCriticalSection2_BeginSlow(c, m1, m2, 0); } } +#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index c461bc1786ddf4..493377b4c25040 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -904,7 +904,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest_size)); @@ -979,7 +978,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_event_loop)); @@ -1055,7 +1053,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_node)); @@ -1071,7 +1068,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(line_buffering)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lineno)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(listcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(little)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locale)); @@ -1218,7 +1214,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(server_hostname)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(server_side)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(session)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setpgroup)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(setsigdef)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 72c2051bd97660..5dfea2f479d5fe 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -395,7 +395,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) STRUCT_FOR_ID(dict) - STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(digest) STRUCT_FOR_ID(digest_size) @@ -470,7 +469,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) STRUCT_FOR_ID(generation) - STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) STRUCT_FOR_ID(get_event_loop) @@ -546,7 +544,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(kwdefaults) STRUCT_FOR_ID(label) - STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) STRUCT_FOR_ID(last_node) @@ -562,7 +559,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(line) STRUCT_FOR_ID(line_buffering) STRUCT_FOR_ID(lineno) - STRUCT_FOR_ID(listcomp) STRUCT_FOR_ID(little) STRUCT_FOR_ID(lo) STRUCT_FOR_ID(locale) @@ -709,7 +705,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(server_hostname) STRUCT_FOR_ID(server_side) STRUCT_FOR_ID(session) - STRUCT_FOR_ID(setcomp) STRUCT_FOR_ID(setpgroup) STRUCT_FOR_ID(setsid) STRUCT_FOR_ID(setsigdef) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f1f427d99dea69..542a75617b4d3c 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -73,6 +73,7 @@ struct trampoline_api_st { int (*free_state)(void* state); void *state; Py_ssize_t code_padding; + Py_ssize_t code_alignment; }; #endif @@ -129,8 +130,6 @@ struct _atexit_runtime_state { //################### // interpreter atexit -typedef void (*atexit_datacallbackfunc)(void *); - typedef struct atexit_callback { atexit_datacallbackfunc func; void *data; diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 32b60cc33a21f1..c4e007e744ce0f 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -13,7 +13,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h +//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/pylock.h #define _Py_HAS_PARKED 2 #define _Py_ONCE_INITIALIZED 4 @@ -25,13 +25,6 @@ PyMutex_LockFast(PyMutex *m) return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Checks if the mutex is currently locked. -static inline int -PyMutex_IsLocked(PyMutex *m) -{ - return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; -} - // Re-initializes the mutex after a fork to the unlocked state. static inline void _PyMutex_at_fork_reinit(PyMutex *m) @@ -51,6 +44,11 @@ typedef enum _PyLockFlags { // Fail if interrupted by a signal while waiting on the lock. _PY_FAIL_IF_INTERRUPTED = 4, + + // Locking & unlocking this lock requires attached thread state. + // If locking returns PY_LOCK_FAILURE, a Python exception *may* be raised. + // (Intended for use with _PY_LOCK_HANDLE_SIGNALS and _PY_LOCK_DETACH.) + _PY_LOCK_PYTHONLOCK = 8, } _PyLockFlags; // Lock a mutex with an optional timeout and additional options. See diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 50807e68e9a70c..40f8ca68c00b72 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -614,7 +614,7 @@ static inline PyObject * _Py_XGetRef(PyObject **ptr) { for (;;) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -629,7 +629,7 @@ _Py_XGetRef(PyObject **ptr) static inline PyObject * _Py_TryXGetRef(PyObject **ptr) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -918,7 +918,7 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); -extern int _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, +PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, PyObject *name, _PyStackRef *method); // Cache the provided init method in the specialization cache of type if the @@ -1033,6 +1033,20 @@ enum _PyAnnotateFormat { int _PyObject_SetDict(PyObject *obj, PyObject *value); +#ifndef Py_GIL_DISABLED +static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) +{ + assert(!_Py_IsStaticImmortal(op)); + op->ob_refcnt++; + _Py_INCREF_STAT_INC(); +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) + if (!_Py_IsImmortal(op)) { + _Py_INCREF_IncRefTotal(); + } +#endif +} +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6e89ca33e4208c..8faf7a4d403f84 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -41,6 +41,7 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); +extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp); /* Various internal finalizers */ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 633e5cf77db918..ea3dfbd2eef9c1 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pythonrun.h" // _PyOS_STACK_MARGIN_SHIFT #include "pycore_typedefs.h" // _PyRuntimeState #include "pycore_tstate.h" @@ -325,7 +326,7 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, PYOS_STACK_MARGIN_SHIFT); + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); } #ifdef __cplusplus diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 7daed1326af8d5..c2832098ddb3e7 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,6 +33,28 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); + +/* Stack size, in "pointers". This must be large enough, so + * no two calls to check recursion depth are more than this far + * apart. In practice, that means it must be larger than the C + * stack consumption of PyEval_EvalDefault */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) +# define _PyOS_LOG2_STACK_MARGIN 12 +#elif defined(Py_DEBUG) && defined(WIN32) +# define _PyOS_LOG2_STACK_MARGIN 12 +#else +# define _PyOS_LOG2_STACK_MARGIN 11 +#endif +#define _PyOS_STACK_MARGIN (1 << _PyOS_LOG2_STACK_MARGIN) +#define _PyOS_STACK_MARGIN_BYTES (_PyOS_STACK_MARGIN * sizeof(void *)) + +#if SIZEOF_VOID_P == 8 +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 3) +#else +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) +#endif + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index d378fcae26cf35..85ced09d29d338 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -902,7 +902,6 @@ extern "C" { INIT_ID(deterministic), \ INIT_ID(device), \ INIT_ID(dict), \ - INIT_ID(dictcomp), \ INIT_ID(difference_update), \ INIT_ID(digest), \ INIT_ID(digest_size), \ @@ -977,7 +976,6 @@ extern "C" { INIT_ID(func), \ INIT_ID(future), \ INIT_ID(generation), \ - INIT_ID(genexpr), \ INIT_ID(get), \ INIT_ID(get_debug), \ INIT_ID(get_event_loop), \ @@ -1053,7 +1051,6 @@ extern "C" { INIT_ID(kw2), \ INIT_ID(kwdefaults), \ INIT_ID(label), \ - INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ INIT_ID(last_node), \ @@ -1069,7 +1066,6 @@ extern "C" { INIT_ID(line), \ INIT_ID(line_buffering), \ INIT_ID(lineno), \ - INIT_ID(listcomp), \ INIT_ID(little), \ INIT_ID(lo), \ INIT_ID(locale), \ @@ -1216,7 +1212,6 @@ extern "C" { INIT_ID(server_hostname), \ INIT_ID(server_side), \ INIT_ID(session), \ - INIT_ID(setcomp), \ INIT_ID(setpgroup), \ INIT_ID(setsid), \ INIT_ID(setsigdef), \ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 10e7199269eb40..6bf82d8322f508 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -626,7 +626,7 @@ _PyStackRef_FromPyObjectNew(PyObject *obj) if (_Py_IsImmortal(obj)) { return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; } - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -637,7 +637,7 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj) { assert(obj != NULL); - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -654,14 +654,14 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ - (PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) + (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) #else static inline _PyStackRef PyStackRef_DUP(_PyStackRef ref) { assert(!PyStackRef_IsNull(ref)); if (PyStackRef_RefcountOnObject(ref)) { - Py_INCREF_MORTAL(BITS_TO_PTR(ref)); + _Py_INCREF_MORTAL(BITS_TO_PTR(ref)); } return ref; } @@ -829,7 +829,7 @@ _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) static inline int _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) { - PyObject *op = _Py_atomic_load_ptr_relaxed(src); + PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src)); if (op == NULL) { *out = PyStackRef_NULL; return 1; diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index e516211f6c6cbc..6018d98d156a65 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1368,10 +1368,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dictcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(difference_update); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1668,10 +1664,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(genexpr); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1972,10 +1964,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(lambda); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(last); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2036,10 +2024,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(listcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(little); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2624,10 +2608,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(setcomp); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(setpgroup); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index 950aa0af290056..4ed8928c0b92a8 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -29,6 +29,12 @@ extern "C" { PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) #define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + PyObject_ClearWeakRefs(obj); \ + } while (0) + #else #define LOCK_WEAKREFS(obj) @@ -37,6 +43,14 @@ extern "C" { #define LOCK_WEAKREFS_FOR_WR(wr) #define UNLOCK_WEAKREFS_FOR_WR(wr) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + if (weakref_list != NULL) { \ + PyObject_ClearWeakRefs(obj); \ + } \ + } while (0) + #endif static inline int _is_dead(PyObject *obj) diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 2a17c891dda811..17634a93f8fa6f 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*); PyAPI_DATA(PyTypeObject) PyModuleDef_Type; #endif +#ifndef _Py_OPAQUE_PYOBJECT typedef struct PyModuleDef_Base { PyObject_HEAD /* The function used to re-initialize the module. @@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base { 0, /* m_index */ \ _Py_NULL, /* m_copy */ \ } +#endif // _Py_OPAQUE_PYOBJECT #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 /* New in 3.5 */ @@ -104,6 +106,8 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif + +#ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; @@ -115,6 +119,7 @@ struct PyModuleDef { inquiry m_clear; freefunc m_free; }; +#endif // _Py_OPAQUE_PYOBJECT #ifdef __cplusplus } diff --git a/Include/object.h b/Include/object.h index c75e9db0cbd935..b1bcc9481871b4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -56,6 +56,11 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif +#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API) +# error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API" +#endif + +#ifndef _Py_OPAQUE_PYOBJECT /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -99,6 +104,8 @@ whose size is determined when the object is allocated. * not necessarily a byte count. */ #define PyObject_VAR_HEAD PyVarObject ob_base; +#endif // !defined(_Py_OPAQUE_PYOBJECT) + #define Py_INVALID_SIZE (Py_ssize_t)-1 /* PyObjects are given a minimum alignment so that the least significant bits @@ -112,7 +119,9 @@ whose size is determined when the object is allocated. * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ -#ifndef Py_GIL_DISABLED +#ifdef _Py_OPAQUE_PYOBJECT + /* PyObject is opaque */ +#elif !defined(Py_GIL_DISABLED) struct _object { #if (defined(__GNUC__) || defined(__clang__)) \ && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) @@ -168,15 +177,18 @@ struct _object { Py_ssize_t ob_ref_shared; // shared (atomic) reference count PyTypeObject *ob_type; }; -#endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) -typedef struct { +#ifndef _Py_OPAQUE_PYOBJECT +struct PyVarObject { PyObject ob_base; Py_ssize_t ob_size; /* Number of items in variable part */ -} PyVarObject; +}; +#endif +typedef struct PyVarObject PyVarObject; /* Cast argument to PyVarObject* type. */ #define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op)) @@ -286,6 +298,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; +#ifndef _Py_OPAQUE_PYOBJECT // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { assert(Py_TYPE(ob) != &PyLong_Type); @@ -295,6 +308,7 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; @@ -304,6 +318,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -323,6 +338,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* diff --git a/Include/lock.h b/Include/pylock.h similarity index 87% rename from Include/lock.h rename to Include/pylock.h index 782b9dbc70d056..1939ef269d3090 100644 --- a/Include/lock.h +++ b/Include/pylock.h @@ -6,7 +6,7 @@ extern "C" { #ifndef Py_LIMITED_API # define Py_CPYTHON_LOCK_H -# include "cpython/lock.h" +# include "cpython/pylock.h" # undef Py_CPYTHON_LOCK_H #endif diff --git a/Include/pyport.h b/Include/pyport.h index 0675294d5bc3b1..89829373be2ca2 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -667,25 +667,6 @@ extern "C" { #endif -// _Py_NO_SANITIZE_UNDEFINED(): Disable Undefined Behavior sanitizer (UBsan) -// on a function. -// -// Clang and GCC 9.0+ use __attribute__((no_sanitize("undefined"))). -// GCC 4.9+ uses __attribute__((no_sanitize_undefined)). -#if defined(__has_feature) -# if __has_feature(undefined_behavior_sanitizer) -# define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) -# endif -#endif -#if !defined(_Py_NO_SANITIZE_UNDEFINED) && defined(__GNUC__) \ - && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9)) -# define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) -#endif -#ifndef _Py_NO_SANITIZE_UNDEFINED -# define _Py_NO_SANITIZE_UNDEFINED -#endif - - // _Py_NONSTRING: The nonstring variable attribute specifies that an object or // member declaration with type array of char, signed char, or unsigned char, // or pointer to such a type is intended to store character arrays that do not diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 716c4ab3c81cae..92b50aa807bf70 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -21,36 +21,15 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); /* Stuff with no proper home (yet) */ PyAPI_DATA(int) (*PyOS_InputHook)(void); -/* Stack size, in "pointers". This must be large enough, so - * no two calls to check recursion depth are more than this far - * apart. In practice, that means it must be larger than the C - * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) -# define PYOS_LOG2_STACK_MARGIN 12 -#else -# define PYOS_LOG2_STACK_MARGIN 11 -#endif -#define PYOS_STACK_MARGIN (1 << PYOS_LOG2_STACK_MARGIN) -#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *)) - -#if SIZEOF_VOID_P == 8 -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 3) -#else -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 2) -#endif - - #if defined(WIN32) -#define USE_STACKCHECK +# define USE_STACKCHECK #endif - #ifdef USE_STACKCHECK /* Check that we aren't overflowing our stack */ PyAPI_FUNC(int) PyOS_CheckStack(void); #endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYTHONRUN_H # include "cpython/pythonrun.h" diff --git a/Include/refcount.h b/Include/refcount.h index 034c453f449f5b..ba34461fefcbb0 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -30,7 +30,7 @@ increase and decrease the objects reference count. In order to offer sufficient resilience to C extensions using the stable ABI compiled against 3.11 or earlier, we set the initial value near the -middle of the range (2**31, 2**32). That way the the refcount can be +middle of the range (2**31, 2**32). That way the refcount can be off by ~1 billion without affecting immortality. Reference count increases will use saturated arithmetic, taking advantage of @@ -117,6 +117,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #endif #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) @@ -140,6 +141,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) #endif } #define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) +#endif // !defined(_Py_OPAQUE_PYOBJECT) // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); @@ -244,20 +246,6 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -#ifndef Py_GIL_DISABLED -static inline Py_ALWAYS_INLINE void Py_INCREF_MORTAL(PyObject *op) -{ - assert(!_Py_IsStaticImmortal(op)); - op->ob_refcnt++; - _Py_INCREF_STAT_INC(); -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) - if (!_Py_IsImmortal(op)) { - _Py_INCREF_IncRefTotal(); - } -#endif -} -#endif - static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 4da6cd47dc859e..9c35684c945b3e 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -286,7 +286,7 @@ object, the GC does not process it twice. Notice that an object that was marked as "tentatively unreachable" and was later moved back to the reachable list will be visited again by the garbage collector -as now all the references that that object has need to be processed as well. This +as now all the references that the object has need to be processed as well. This process is really a breadth first search over the object graph. Once all the objects are scanned, the GC knows that all container objects in the tentatively unreachable list are really unreachable and can thus be garbage collected. diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index c25066eb107de1..16cf56f62cc1e5 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix): ) self._ftstring_helper(fstring_parts) - def _tstring_helper(self, node): - if not node.values: - self._write_ftstring([], "t") - return - last_idx = 0 - for i, value in enumerate(node.values): - # This can happen if we have an implicit concat of a t-string - # with an f-string - if isinstance(value, FormattedValue): - if i > last_idx: - # Write t-string until here - self._write_ftstring(node.values[last_idx:i], "t") - self.write(" ") - # Write f-string with the current formatted value - self._write_ftstring([node.values[i]], "f") - if i + 1 < len(node.values): - # Only add a space if there are more values after this - self.write(" ") - last_idx = i + 1 - - if last_idx < len(node.values): - # Write t-string from last_idx to end - self._write_ftstring(node.values[last_idx:], "t") - def visit_JoinedStr(self, node): self._write_ftstring(node.values, "f") def visit_TemplateStr(self, node): - self._tstring_helper(node) + self._write_ftstring(node.values, "t") def _write_ftstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 781b38ec26ba33..9b8e42a2342536 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -6122,7 +6122,11 @@ def _convert_for_comparison(self, other, equality_op=False): (?P0)? (?P\d+)? (?P[,_])? -(?:\.(?P\d+))? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) @@ -6215,6 +6219,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6334,6 +6341,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/_pyio.py b/Lib/_pyio.py index fb2a6d049caab6..5db8ce9244b5ba 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -876,16 +876,28 @@ class BytesIO(BufferedIOBase): _buffer = None def __init__(self, initial_bytes=None): + # Use to keep self._buffer and self._pos consistent. + self._lock = Lock() + buf = bytearray() if initial_bytes is not None: buf += initial_bytes - self._buffer = buf - self._pos = 0 + + with self._lock: + self._buffer = buf + self._pos = 0 def __getstate__(self): if self.closed: raise ValueError("__getstate__ on closed file") - return self.__dict__.copy() + with self._lock: + state = self.__dict__.copy() + del state['_lock'] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._lock = Lock() def getvalue(self): """Return the bytes value (contents) of the buffer @@ -918,14 +930,16 @@ def read(self, size=-1): raise TypeError(f"{size!r} is not an integer") else: size = size_index() - if size < 0: - size = len(self._buffer) - if len(self._buffer) <= self._pos: - return b"" - newpos = min(len(self._buffer), self._pos + size) - b = self._buffer[self._pos : newpos] - self._pos = newpos - return bytes(b) + + with self._lock: + if size < 0: + size = len(self._buffer) + if len(self._buffer) <= self._pos: + return b"" + newpos = min(len(self._buffer), self._pos + size) + b = self._buffer[self._pos : newpos] + self._pos = newpos + return bytes(b) def read1(self, size=-1): """This is the same as read. @@ -941,12 +955,14 @@ def write(self, b): n = view.nbytes # Size of any bytes-like object if n == 0: return 0 - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n + + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = b + self._pos += n return n def seek(self, pos, whence=0): @@ -963,9 +979,11 @@ def seek(self, pos, whence=0): raise ValueError("negative seek position %r" % (pos,)) self._pos = pos elif whence == 1: - self._pos = max(0, self._pos + pos) + with self._lock: + self._pos = max(0, self._pos + pos) elif whence == 2: - self._pos = max(0, len(self._buffer) + pos) + with self._lock: + self._pos = max(0, len(self._buffer) + pos) else: raise ValueError("unsupported whence value") return self._pos @@ -978,18 +996,20 @@ def tell(self): def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") - if pos is None: - pos = self._pos - else: - try: - pos_index = pos.__index__ - except AttributeError: - raise TypeError(f"{pos!r} is not an integer") + + with self._lock: + if pos is None: + pos = self._pos else: - pos = pos_index() - if pos < 0: - raise ValueError("negative truncate position %r" % (pos,)) - del self._buffer[pos:] + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f"{pos!r} is not an integer") + else: + pos = pos_index() + if pos < 0: + raise ValueError("negative truncate position %r" % (pos,)) + del self._buffer[pos:] return pos def readable(self): diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50ac7..00000000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 8956fb1242e52a..e0535d50396316 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -27,6 +27,7 @@ import linecache from dataclasses import dataclass, field import os.path +import re import sys @@ -195,7 +196,19 @@ def runsource(self, source, filename="", symbol="single"): ast.PyCF_ONLY_AST, incomplete_input=False, ) - except (SyntaxError, OverflowError, ValueError): + except SyntaxError as e: + # If it looks like pip install was entered (a common beginner + # mistake), provide a hint to use the system command prompt. + if re.match(r"^\s*(pip3?|py(thon3?)? -m pip) install.*", source): + e.add_note( + "The Python package manager (pip) can only be used" + " outside of the Python REPL.\n" + "Try the 'pip' command in a separate terminal or" + " command prompt." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): self.showsyntaxerror(filename, source=source) return False if tree.body: diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f6835d1..00000000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 9560ae779abfea..23b8fa6b9c7625 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -43,10 +43,11 @@ Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 00000000000000..d02ef69cce0bd8 --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,488 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Populate the _capabilities dict for easy retrieval + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + + # Read string offsets + end_offset = offset + 2 * str_count + if offset > len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): + if off < 0: + # CANCELLED_STRING; we do not store those + continue + elif off < len(string_table): + # Find null terminator + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + + self._capabilities = capabilities + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + return self._capabilities.get(cap) + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3cce..943ee12f964b29 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index d21cdd9b076d86..a7e49923191c07 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,7 +33,7 @@ import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event from .fancy_termios import tcgetattr, tcsetattr from .trace import trace @@ -60,7 +60,7 @@ class InvalidTerminal(RuntimeError): pass -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) SIGWINCH_EVENT = "repaint" @@ -157,7 +157,7 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term @overload @@ -167,7 +167,7 @@ def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -201,7 +201,7 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) + self.event_queue = EventQueue(self.input_fd, self.encoding, self.terminfo) self.cursor_visible = 1 signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -597,14 +597,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -701,7 +701,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 29b3e9dd5efd07..2a9cca59e7477f 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,7 +18,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from . import curses +from .terminfo import TermInfo from .trace import trace from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE @@ -54,22 +54,23 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes + class EventQueue(BaseEventQueue): - def __init__(self, fd: int, encoding: str) -> None: - keycodes = get_terminal_keycodes() + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] keycodes[backspace] = "backspace" diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index e04fbdc6c8a5c4..fd788c8429e15b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -241,14 +241,14 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return s in keyword_first_sets_match return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) | TI(T.OP, string="(" | "*" | "-" | "[" | "{") ): return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NAME, string=s) ): diff --git a/Lib/argparse.py b/Lib/argparse.py index 83258cf3e0f37d..2144c81886ad19 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -167,7 +167,7 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=False, + color=True, ): # default setting for width if width is None: @@ -1231,7 +1231,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() - self._color = False + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1878,7 +1878,7 @@ def __init__(self, exit_on_error=True, *, suggest_on_error=False, - color=False, + color=True, ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 4be7112fa017d4..32a5dbae03af21 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -51,15 +51,24 @@ def __getattr__(name: str): import warnings - deprecated = { - "AbstractEventLoopPolicy", - "DefaultEventLoopPolicy", - "WindowsSelectorEventLoopPolicy", - "WindowsProactorEventLoopPolicy", - } - if name in deprecated: - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - # deprecated things have underscores in front of them - return globals()["_" + name] + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 21ca5c5f62ae83..ff3a69d1e17297 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -64,7 +64,7 @@ def callback(): except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback, context=self.context) + self.loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 04fb961e9985e4..520d4b398545bf 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1016,38 +1016,43 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): family, type_, proto, _, address = addr_info sock = None try: - sock = socket.socket(family=family, type=type_, proto=proto) - sock.setblocking(False) - if local_addr_infos is not None: - for lfamily, _, _, _, laddr in local_addr_infos: - # skip local addresses of different family - if lfamily != family: - continue - try: - sock.bind(laddr) - break - except OSError as exc: - msg = ( - f'error while attempting to bind on ' - f'address {laddr!r}: {str(exc).lower()}' - ) - exc = OSError(exc.errno, msg) - my_exceptions.append(exc) - else: # all bind attempts failed - if my_exceptions: - raise my_exceptions.pop() - else: - raise OSError(f"no matching local address with {family=} found") - await self.sock_connect(sock, address) - return sock - except OSError as exc: - my_exceptions.append(exc) - if sock is not None: - sock.close() - raise + try: + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: {str(exc).lower()}' + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await self.sock_connect(sock, address) + return sock + except OSError as exc: + my_exceptions.append(exc) + raise except: if sock is not None: - sock.close() + try: + sock.close() + except OSError: + # An error when closing a newly created socket is + # not important, but it can overwrite more important + # non-OSError error. So ignore it. + pass raise finally: exceptions = my_exceptions = None @@ -1161,7 +1166,7 @@ async def create_connection( raise ExceptionGroup("create_connection failed", exceptions) if len(exceptions) == 1: raise exceptions[0] - else: + elif exceptions: # If they all have the same str(), raise one. model = str(exceptions[0]) if all(str(exc) == model for exc in exceptions): @@ -1170,6 +1175,9 @@ async def create_connection( # the various error messages. raise OSError('Multiple exceptions: {}'.format( ', '.join(str(exc) for exc in exceptions))) + else: + # No exceptions were collected, raise a timeout error + raise TimeoutError('create_connection failed') finally: exceptions = None diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2913f901dca65f..a7fb55982abe9c 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - "_AbstractEventLoopPolicy", "AbstractEventLoop", "AbstractServer", "Handle", "TimerHandle", - "_get_event_loop_policy", "get_event_loop_policy", - "_set_event_loop_policy", "set_event_loop_policy", "get_event_loop", "set_event_loop", @@ -791,7 +788,10 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import _DefaultEventLoopPolicy + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy _event_loop_policy = _DefaultEventLoopPolicy() diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 2f3865114a84f9..084fccaaff2ff7 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -227,9 +227,6 @@ def task_done(self): been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises ValueError if called more times than there were items placed in the queue. """ @@ -256,9 +253,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. """ self._is_shutdown = True if immediate: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f69c6a64c39ae6..1c1458127db5ac 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,6 @@ __all__ = ( 'SelectorEventLoop', - '_DefaultEventLoopPolicy', 'EventLoop', ) diff --git a/Lib/bdb.py b/Lib/bdb.py index 4290ef22302a42..efc3e0a235ac8e 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -267,12 +267,9 @@ def trace_dispatch(self, frame, event, arg): is entered. return: A function or other code block is about to return. exception: An exception has occurred. - c_call: A C function is about to be called. - c_return: A C function has returned. - c_exception: A C function has raised an exception. - For the Python events, specialized functions (see the dispatch_*() - methods) are called. For the C events, no action is taken. + For all the events, specialized functions (see the dispatch_*() + methods) are called. The arg parameter depends on the previous event. """ @@ -288,12 +285,6 @@ def trace_dispatch(self, frame, event, arg): return self.dispatch_return(frame, arg) if event == 'exception': return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': - return self.trace_dispatch if event == 'opcode': return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) diff --git a/Lib/codecs.py b/Lib/codecs.py index fc38e922257644..e4a8010aba90a5 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -618,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -709,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d2ddc1cd9ec2ea..b8653f40a942f0 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -776,23 +776,26 @@ def __repr__(self): # When the multiplicities are all zero or one, multiset operations # are guaranteed to be equivalent to the corresponding operations # for regular sets. + # # Given counter multisets such as: # cp = Counter(a=1, b=0, c=1) # cq = Counter(c=1, d=0, e=1) + # # The corresponding regular sets would be: # sp = {'a', 'c'} # sq = {'c', 'e'} + # # All of the following relations would hold: - # set(cp + cq) == sp | sq - # set(cp - cq) == sp - sq - # set(cp | cq) == sp | sq - # set(cp & cq) == sp & sq # (cp == cq) == (sp == sq) # (cp != cq) == (sp != sq) # (cp <= cq) == (sp <= sq) # (cp < cq) == (sp < sq) # (cp >= cq) == (sp >= sq) # (cp > cq) == (sp > sq) + # set(cp + cq) == sp | sq + # set(cp - cq) == sp - sq + # set(cp | cq) == sp | sq + # set(cp & cq) == sp & sq def __eq__(self, other): 'True if all counts agree. Missing counts are treated as zero.' diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 7ada7431c1ab8c..e717222cf98b32 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,7 +17,7 @@ wait, as_completed) -__all__ = ( +__all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', @@ -29,10 +29,18 @@ 'Executor', 'wait', 'as_completed', - 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', -) +] + + +try: + import _interpreters +except ImportError: + _interpreters = None + +if _interpreters: + __all__.append('InterpreterPoolExecutor') def __dir__(): @@ -43,22 +51,15 @@ def __getattr__(name): global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor as pe - ProcessPoolExecutor = pe - return pe + from .process import ProcessPoolExecutor + return ProcessPoolExecutor if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor as te - ThreadPoolExecutor = te - return te + from .thread import ThreadPoolExecutor + return ThreadPoolExecutor - if name == 'InterpreterPoolExecutor': - try: - from .interpreter import InterpreterPoolExecutor as ie - except ModuleNotFoundError: - ie = InterpreterPoolExecutor = None - else: - InterpreterPoolExecutor = ie - return ie + if _interpreters and name == 'InterpreterPoolExecutor': + from .interpreter import InterpreterPoolExecutor + return InterpreterPoolExecutor raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index cbb60ce80c1813..53c6e757ded2e3 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -118,5 +118,7 @@ def __init__(self, max_workers=None, thread_name_prefix='', each worker interpreter. initargs: A tuple of arguments to pass to the initializer. """ + thread_name_prefix = (thread_name_prefix or + (f"InterpreterPoolExecutor-{self._counter()}")) super().__init__(max_workers, thread_name_prefix, initializer, initargs) diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index 0fd661249a276c..aa46a2b37a48d5 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -146,12 +146,8 @@ def __del__(self): self._decref() # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (self._id,)) def _decref(self): if not self._ownsref: diff --git a/Lib/concurrent/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py index f47eb693ac861c..a5f46b20fbb4c5 100644 --- a/Lib/concurrent/interpreters/_crossinterp.py +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -40,16 +40,21 @@ class UnboundItem: @classonly def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) + doc = cls.__doc__ + if doc: + doc = doc.replace( + 'cross-interpreter container', kind, + ).replace( + 'cross-interpreter', kind, + ) subclass = type( f'Unbound{kind.capitalize()}Item', (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), + { + "_MODULE": module, + "_NAME": name, + "__doc__": doc, + }, ) return object.__new__(subclass) diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index 99987f2f6926b0..9c12b2c8c24664 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -129,12 +129,8 @@ def __hash__(self): return hash(self._id) # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (self._id,)) def _set_unbound(self, op, items=None): assert not hasattr(self, '_unbound') diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 86d29df0639184..83ea623dce6281 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1265,7 +1265,7 @@ def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): doc = getattr(defined_fields.get(slot), 'doc', None) if doc is not None: seen_docs = True - slots.update({slot: doc}) + slots[slot] = doc # We only return dict if there's at least one doc member, # otherwise we return tuple, which is the old default format. diff --git a/Lib/doctest.py b/Lib/doctest.py index c8c95ecbb273b2..92a2ab4f7e66f8 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -1141,7 +1142,9 @@ def _find_lineno(self, obj, source_lines): if inspect.ismethod(obj): obj = obj.__func__ if isinstance(obj, property): obj = obj.fget - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. obj = inspect.unwrap(obj) try: @@ -1997,8 +2000,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. diff --git a/Lib/email/message.py b/Lib/email/message.py index 41fcc2b9778798..36e4b4a9f0b773 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -313,6 +313,8 @@ def get_payload(self, i=None, decode=False): # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 60a8d5eb227f82..0c90b4c9fe18fa 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -316,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling: {errors}") + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return ("", 0) diff --git a/Lib/enum.py b/Lib/enum.py index 01fecca3e5aac0..c00ae85d2f8efe 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -535,7 +535,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) # - # Flag structures (will be removed if final class is not a Flag + # Flag structures (will be removed if final class is not a Flag) classdict['_boundary_'] = ( boundary or getattr(first_enum, '_boundary_', None) @@ -544,6 +544,29 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_singles_mask_'] = 0 classdict['_all_bits_'] = 0 classdict['_inverted_'] = None + # check for negative flag values and invert if found (using _proto_members) + if Flag is not None and bases and issubclass(bases[-1], Flag): + bits = 0 + inverted = [] + for n in member_names: + p = classdict[n] + if isinstance(p.value, int): + if p.value < 0: + inverted.append(p) + else: + bits |= p.value + elif p.value is None: + pass + elif isinstance(p.value, tuple) and p.value and isinstance(p.value[0], int): + if p.value[0] < 0: + inverted.append(p) + else: + bits |= p.value[0] + for p in inverted: + if isinstance(p.value, int): + p.value = bits & p.value + else: + p.value = (bits & p.value[0], ) + p.value[1:] try: classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) @@ -1487,7 +1510,10 @@ def _missing_(cls, value): ) if value < 0: neg_value = value - value = all_bits + 1 + value + if cls._boundary_ in (EJECT, KEEP): + value = all_bits + 1 + value + else: + value = singles_mask & value # get members and unknown unknown = value & ~flag_mask aliases = value & ~singles_mask @@ -1965,7 +1991,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) diff --git a/Lib/fractions.py b/Lib/fractions.py index cb05ae7c2003f0..c1b12e7a1c091c 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -170,7 +170,11 @@ def _round_to_figures(n, d, figures): (?P0(?=[0-9]))? (?P[0-9]+)? (?P[,_])? - (?:\.(?P[0-9]+))? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch @@ -499,11 +503,15 @@ def _format_float_style(self, match): minimumwidth = int(match["minimumwidth"] or "0") thousands_sep = match["thousands_sep"] precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" presentation_type = match["presentation_type"] trim_zeros = presentation_type in "gG" and not alternate_form trim_point = not alternate_form exponent_indicator = "E" if presentation_type in "EFG" else "e" + if align == '=' and fill == '0': + zeropad = True + # Round to get the digits we need, figure out where to place the point, # and decide whether to use scientific notation. 'point_pos' is the # relative to the _end_ of the digit string: that is, it's the number @@ -552,6 +560,9 @@ def _format_float_style(self, match): if trim_zeros: frac_part = frac_part.rstrip("0") separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) trailing = separator + frac_part + suffix # Do zero padding if required. diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 9363f564aab7a6..4a223654994194 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -81,28 +81,28 @@ def isdevdrive(path): return False -def getsize(filename): +def getsize(filename, /): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size -def getmtime(filename): +def getmtime(filename, /): """Return the last modification time of a file, reported by os.stat().""" return os.stat(filename).st_mtime -def getatime(filename): +def getatime(filename, /): """Return the last access time of a file, reported by os.stat().""" return os.stat(filename).st_atime -def getctime(filename): +def getctime(filename, /): """Return the metadata change time of a file, reported by os.stat().""" return os.stat(filename).st_ctime # Return the longest prefix of all list elements. -def commonprefix(m): +def commonprefix(m, /): "Given a list of pathnames, returns the longest common leading component" if not m: return '' # Some people pass in a list of pathname parts to operate in an OS-agnostic @@ -120,14 +120,14 @@ def commonprefix(m): # Are two stat buffers (obtained from stat, fstat or lstat) # describing the same file? -def samestat(s1, s2): +def samestat(s1, s2, /): """Test whether two stat buffers reference the same file""" return (s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev) # Are two filenames really pointing to the same file? -def samefile(f1, f2): +def samefile(f1, f2, /): """Test whether two pathnames reference the same actual file or directory This is determined by the device number and i-node number and diff --git a/Lib/gzip.py b/Lib/gzip.py index c00f51858de0f0..89c4738ec69325 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -30,7 +30,7 @@ _WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE -def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, +def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. @@ -158,7 +158,7 @@ class GzipFile(_streams.BaseStream): myfileobj = None def __init__(self, filename=None, mode=None, - compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None): + compresslevel=_COMPRESS_LEVEL_TRADEOFF, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a @@ -621,7 +621,7 @@ def _rewind(self): self._new_member = True -def compress(data, compresslevel=_COMPRESS_LEVEL_BEST, *, mtime=0): +def compress(data, compresslevel=_COMPRESS_LEVEL_TRADEOFF, *, mtime=0): """Compress data in one shot and return the compressed string. compresslevel sets the compression level in range of 0-9. diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 0e9bd98aa1fc31..a7db778b716537 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -2,7 +2,7 @@ # Licensed to PSF under a Contributor Agreement. # -__doc__ = """hashlib module - A common interface to many hash functions. +__doc__ = r"""hashlib module - A common interface to many hash functions. new(name, data=b'', **kwargs) - returns a new hash object implementing the given hash function; initializing the hash @@ -12,7 +12,7 @@ than using new(name): md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(), -sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256. +sha3_224(), sha3_256(), sha3_384(), sha3_512(), shake_128(), and shake_256(). More algorithms may be available on your platform but the above are guaranteed to exist. See the algorithms_guaranteed and algorithms_available attributes @@ -21,8 +21,8 @@ NOTE: If you want the adler32 or crc32 hash functions they are available in the zlib module. -Choose your hash function wisely. Some have known collision weaknesses. -sha384 and sha512 will be slow on 32 bit platforms. +Choose your hash function wisely. Some have known collision weaknesses, +while others may be slower depending on the CPU architecture. Hash objects have these methods: - update(data): Update the hash object with the bytes in data. Repeated calls @@ -36,20 +36,20 @@ efficiently compute the digests of data that share a common initial substring. -For example, to obtain the digest of the byte string 'Nobody inspects the -spammish repetition': +Assuming that Python has been built with MD5 support, the following computes +the MD5 digest of the byte string b'Nobody inspects the spammish repetition': >>> import hashlib >>> m = hashlib.md5() >>> m.update(b"Nobody inspects") >>> m.update(b" the spammish repetition") >>> m.digest() - b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' + b'\xbbd\x9c\x83\xdd\x1e\xa5\xc9\xd9\xde\xc9\xa1\x8d\xf0\xff\xe9' More condensed: - >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() - 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' + >>> hashlib.md5(b"Nobody inspects the spammish repetition").hexdigest() + 'bb649c83dd1ea5c9d9dec9a18df0ffe9' """ @@ -80,6 +80,11 @@ } def __get_builtin_constructor(name): + if not isinstance(name, str): + # Since this function is only used by new(), we use the same + # exception as _hashlib.new() when 'name' is of incorrect type. + err = f"new() argument 'name' must be str, not {type(name).__name__}" + raise TypeError(err) cache = __builtin_constructor_cache constructor = cache.get(name) if constructor is not None: @@ -120,20 +125,33 @@ def __get_builtin_constructor(name): if constructor is not None: return constructor - raise ValueError('unsupported hash type ' + name) + # Keep the message in sync with hashlib.h::HASHLIB_UNSUPPORTED_ALGORITHM. + raise ValueError(f'unsupported hash algorithm {name}') def __get_openssl_constructor(name): + # This function is only used until the module has been initialized. + assert isinstance(name, str), "invalid call to __get_openssl_constructor()" if name in __block_openssl_constructor: # Prefer our builtin blake2 implementation. return __get_builtin_constructor(name) try: - # MD5, SHA1, and SHA2 are in all supported OpenSSL versions - # SHA3/shake are available in OpenSSL 1.1.1+ + # Fetch the OpenSSL hash function if it exists, + # independently of the context security policy. f = getattr(_hashlib, 'openssl_' + name) - # Allow the C module to raise ValueError. The function will be - # defined but the hash not actually available. Don't fall back to - # builtin if the current security policy blocks a digest, bpo#40695. + # Check if the context security policy blocks the digest or not + # by allowing the C module to raise a ValueError. The function + # will be defined but the hash will not be available at runtime. + # + # We use "usedforsecurity=False" to prevent falling back to the + # built-in function in case the security policy does not allow it. + # + # Note that this only affects the explicit named constructors, + # and not the algorithms exposed through hashlib.new() which + # can still be resolved to a built-in function even if the + # current security policy does not allow it. + # + # See https://github.com/python/cpython/issues/84872. f(usedforsecurity=False) # Use the C function directly (very fast) return f @@ -154,6 +172,8 @@ def __hash_new(name, *args, **kwargs): optionally initialized with data (which must be a bytes-like object). """ if name in __block_openssl_constructor: + # __block_openssl_constructor is expected to contain strings only + assert isinstance(name, str), f"unexpected name: {name}" # Prefer our builtin blake2 implementation. return __get_builtin_constructor(name)(*args, **kwargs) try: @@ -187,7 +207,8 @@ def __hash_new(name, *args, **kwargs): try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt # noqa: F401 + from _hashlib import scrypt + __all__ += ('scrypt',) except ImportError: pass @@ -203,7 +224,7 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): *digest* must either be a hash algorithm name as a *str*, a hash constructor, or a callable that returns a hash object. """ - # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy + # On Linux we could use AF_ALG sockets and sendfile() to achieve zero-copy # hashing with hardware acceleration. if isinstance(digest, str): digestobj = new(digest) diff --git a/Lib/hmac.py b/Lib/hmac.py index 3683a4aa653a0a..e50d355fc60871 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -26,6 +26,16 @@ digest_size = None +def _is_shake_constructor(digest_like): + if isinstance(digest_like, str): + name = digest_like + else: + h = digest_like() if callable(digest_like) else digest_like.new() + if not isinstance(name := getattr(h, "name", None), str): + return False + return name.startswith(("shake", "SHAKE")) + + def _get_digest_constructor(digest_like): if callable(digest_like): return digest_like @@ -109,6 +119,8 @@ def _init_old(self, key, msg, digestmod): import warnings digest_cons = _get_digest_constructor(digestmod) + if _is_shake_constructor(digest_cons): + raise ValueError(f"unsupported hash algorithm {digestmod}") self._hmac = None self._outer = digest_cons() @@ -243,6 +255,8 @@ def digest(key, msg, digest): def _compute_digest_fallback(key, msg, digest): digest_cons = _get_digest_constructor(digest) + if _is_shake_constructor(digest_cons): + raise ValueError(f"unsupported hash algorithm {digest}") inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index ba416e7fa6e3fe..5d03c98df5cdd0 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -29,17 +29,46 @@ starttagopen = re.compile('<[a-zA-Z]') endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -56,8 +85,6 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') # Character reference processing logic specific to attribute values @@ -101,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase): """ CDATA_CONTENT_ELEMENTS = ("script", "style") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") def __init__(self, *, convert_charrefs=True): """Initialize and reset this instance. @@ -118,6 +146,7 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True super().reset() def feed(self, data): @@ -139,13 +168,20 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self._escapable = escapable + if escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -166,7 +202,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -178,7 +214,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -280,7 +316,7 @@ def goahead(self, end): assert 0, "interesting.search() lied" # end while if end and i < n: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:n])) else: self.handle_data(rawdata[i:n]) @@ -309,8 +345,23 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith(' ¬-an-entity-ref;', "", '

', @@ -279,70 +285,223 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) - - def test_cdata_with_closing_tags(self): + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + 'a::before { content: "< /style>"; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /title>', + '', + '', + '', + '', + '', + ]) + def test_title_content(self, content): + source = f"{content}" + self._run_check(source, [ + ("starttag", "title", []), + ("data", content), + ("endtag", "title"), + ]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /textarea>', + '', + '', + '', + '', + ]) + def test_textarea_content(self, content): + source = f"" + self._run_check(source, [ + ("starttag", "textarea", []), + ("data", content), + ("endtag", "textarea"), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """ ¬-an-entity-ref;

''""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'Egg & Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -443,7 +602,7 @@ def test_starttag_junk_chars(self): self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) self._run_check("</a", []) - self._run_check("</ a>", [('endtag', 'a')]) + self._run_check("</ a>", [('comment', ' a')]) self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) @@ -491,6 +650,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -525,15 +688,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -749,9 +908,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -760,6 +925,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -826,13 +995,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 38429ad480ff1c..47e3914d1dd62e 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -386,6 +386,52 @@ def test_headers_debuglevel(self): self.assertEqual(lines[2], "header: Second: val1") self.assertEqual(lines[3], "header: Second: val2") + def test_max_response_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = [f"Name{i}: Value{i}".encode() for i in range(max_headers)] + body = b"HTTP/1.1 200 OK\r\n" + b"\r\n".join(headers) + + with self.subTest(max_headers=None): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + with self.assertRaisesRegex( + client.HTTPException, f"got more than 100 headers" + ): + resp.begin() + + with self.subTest(max_headers=max_headers): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin(_max_headers=max_headers) + + def test_max_connection_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = ( + f"Name{i}: Value{i}".encode() for i in range(max_headers - 1) + ) + body = ( + b"HTTP/1.1 200 OK\r\n" + + b"\r\n".join(headers) + + b"\r\nContent-Length: 12\r\n\r\nDummy body\r\n" + ) + + with self.subTest(max_headers=None): + conn = client.HTTPConnection("example.com") + conn.sock = FakeSocket(body) + conn.request("GET", "/") + with self.assertRaisesRegex( + client.HTTPException, f"got more than {client._MAXHEADERS} headers" + ): + response = conn.getresponse() + + with self.subTest(max_headers=None): + conn = client.HTTPConnection( + "example.com", max_response_headers=max_headers + ) + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + response.read() class HttpMethodTests(TestCase): def test_invalid_method_names(self): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6e34094c5aa422..abbd5f1ed5f12f 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -35,7 +35,7 @@ cpython_only, is_apple_mobile, is_emscripten, - is_wasi, + is_wasm32, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, @@ -1257,7 +1257,7 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 5de89714eb50c7..6d6d5f96aab4a8 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -635,7 +635,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3495 + EXPECTED_MAGIC_NUMBER = 3625 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 79eb103224b956..30e01b8cd87a75 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2820,7 +2820,7 @@ async def asyncTearDown(self): @classmethod def tearDownClass(cls): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) @@ -5786,6 +5786,7 @@ def test_types_module_has_signatures(self): 'AsyncGeneratorType': {'athrow'}, 'CoroutineType': {'throw'}, 'GeneratorType': {'throw'}, + 'FrameLocalsProxyType': {'setdefault', 'pop', 'get'}, } self._test_module_has_signatures(types, unsupported_signature=unsupported_signature, @@ -5916,6 +5917,7 @@ def test_sysconfig_module_has_signatures(self): def test_threading_module_has_signatures(self): import threading self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) def test_thread_module_has_signatures(self): import _thread diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 0ee4582b5d1568..a34b20beaca7a3 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -412,9 +412,11 @@ def test_equality(self): def test_pickle(self): interp = interpreters.create() - data = pickle.dumps(interp) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, interp) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(interp, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) class TestInterpreterIsRunning(TestBase): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 109ddf344539ad..52827357078b85 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -121,9 +121,11 @@ def test_equality(self): def test_pickle(self): ch, _ = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendChannelAttrs(TestBase): @@ -152,9 +154,11 @@ def test_equality(self): def test_pickle(self): _, ch = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendRecv(TestBase): diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index cb17340f581b0a..5451c6654acb47 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -188,9 +188,11 @@ def test_equality(self): def test_pickle(self): queue = queues.create() - data = pickle.dumps(queue) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, queue) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(queue, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) class TestQueueOps(TestBase): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 0c921ffbc2576a..b487bcabf01ca4 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -9,6 +9,7 @@ # * test_univnewlines - tests universal newline support # * test_largefile - tests operations on a file greater than 2**32 bytes # (only enabled with -ulargefile) +# * test_free_threading/test_io - tests thread safety of io objects ################################################################################ # ATTENTION TEST WRITERS!!! diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 1b9f3cf76240ad..18e4b676c53236 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -1147,7 +1147,7 @@ def test_error_iter(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index cffdeeacc5d73b..70148dc30fc957 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -716,7 +716,7 @@ def test_multiple_comprehension_name_reuse(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 3819965ed2cbf9..275f7ce47d09b5 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5421,7 +5421,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5433,7 +5433,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5758,7 +5758,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) if handler: handler.close() diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 46cb54647b1968..e3b0d4fa9eeeb3 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -17,6 +17,7 @@ eps = 1E-05 NAN = float('nan') +NNAN = float('-nan') INF = float('inf') NINF = float('-inf') FLOAT_MAX = sys.float_info.max @@ -636,6 +637,92 @@ def testFmod(self): self.assertEqual(math.fmod(0.0, NINF), 0.0) self.assertRaises(ValueError, math.fmod, INF, INF) + def test_fmax(self): + self.assertRaises(TypeError, math.fmax) + self.assertRaises(TypeError, math.fmax, 'x', 'y') + + self.assertEqual(math.fmax(0., 0.), 0.) + self.assertEqual(math.fmax(1., 2.), 2.) + self.assertEqual(math.fmax(2., 1.), 2.) + + self.assertEqual(math.fmax(+1., +0.), 1.) + self.assertEqual(math.fmax(+0., +1.), 1.) + self.assertEqual(math.fmax(+1., -0.), 1.) + self.assertEqual(math.fmax(-0., +1.), 1.) + + self.assertEqual(math.fmax(-1., +0.), 0.) + self.assertEqual(math.fmax(+0., -1.), 0.) + self.assertEqual(math.fmax(-1., -0.), 0.) + self.assertEqual(math.fmax(-0., -1.), 0.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmax(INF, x), INF) + self.assertEqual(math.fmax(x, INF), INF) + self.assertEqual(math.fmax(NINF, x), x) + self.assertEqual(math.fmax(x, NINF), x) + + @requires_IEEE_754 + def test_fmax_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmax(NAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NAN))) + self.assertFalse(math.isnan(math.fmax(NNAN, x))) + self.assertFalse(math.isnan(math.fmax(x, NNAN))) + # When both operands are NaNs, fmax() returns NaN (see C11, F.10.9.2) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmax(NAN, NAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmax(NNAN, NAN))) + + def test_fmin(self): + self.assertRaises(TypeError, math.fmin) + self.assertRaises(TypeError, math.fmin, 'x', 'y') + + self.assertEqual(math.fmin(0., 0.), 0.) + self.assertEqual(math.fmin(1., 2.), 1.) + self.assertEqual(math.fmin(2., 1.), 1.) + + self.assertEqual(math.fmin(+1., +0.), 0.) + self.assertEqual(math.fmin(+0., +1.), 0.) + self.assertEqual(math.fmin(+1., -0.), 0.) + self.assertEqual(math.fmin(-0., +1.), 0.) + + self.assertEqual(math.fmin(-1., +0.), -1.) + self.assertEqual(math.fmin(+0., -1.), -1.) + self.assertEqual(math.fmin(-1., -0.), -1.) + self.assertEqual(math.fmin(-0., -1.), -1.) + + for x in [NINF, -1., -0., 0., 1., INF]: + self.assertFalse(math.isnan(x)) + + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertEqual(math.fmin(INF, x), x) + self.assertEqual(math.fmin(x, INF), x) + self.assertEqual(math.fmin(NINF, x), NINF) + self.assertEqual(math.fmin(x, NINF), NINF) + + @requires_IEEE_754 + def test_fmin_nans(self): + # When exactly one operand is NaN, the other is returned. + for x in [NINF, -1., -0., 0., 1., INF]: + with self.subTest(x=x, is_negative=math.copysign(1, x) < 0): + self.assertFalse(math.isnan(math.fmin(NAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NAN))) + self.assertFalse(math.isnan(math.fmin(NNAN, x))) + self.assertFalse(math.isnan(math.fmin(x, NNAN))) + # When both operands are NaNs, fmin() returns NaN (see C11, F.10.9.3) + # whose sign is implementation-defined (see C11, F.10.0.3). + self.assertTrue(math.isnan(math.fmin(NAN, NAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NAN, NNAN))) + self.assertTrue(math.isnan(math.fmin(NNAN, NAN))) + def testFrexp(self): self.assertRaises(TypeError, math.frexp) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index fd4197b7086976..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -732,7 +732,7 @@ def test_tagname(self): m2.close() m1.close() - with self.assertRaisesRegex(TypeError, 'must be str or None'): + with self.assertRaisesRegex(TypeError, 'tagname'): mmap.mmap(-1, 8, tagname=1) @cpython_only diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 5217037ae9d812..de3a17fe893170 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -104,7 +104,7 @@ def create_file(filename, content=b'content'): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MiscTests(unittest.TestCase): @@ -1918,11 +1918,9 @@ def test_makedir(self): support.is_wasi, "WASI's umask is a stub." ) - @unittest.skipIf( - support.is_emscripten, - "TODO: Fails in buildbot; see #135783" - ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index b2e2cdb3338beb..a1105aae6351b6 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -17,7 +17,7 @@ from test.support import import_helper from test.support import cpython_only -from test.support import is_emscripten, is_wasi +from test.support import is_emscripten, is_wasi, is_wasm32 from test.support import infinite_recursion from test.support import os_helper from test.support.os_helper import TESTFN, FS_NONASCII, FakePath @@ -539,12 +539,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_is_reserved_deprecated(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_reserved() - def test_full_match_case_sensitive(self): P = self.cls self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) @@ -3164,7 +3158,7 @@ def test_absolute_posix(self): self.assertEqual(str(P('//a/b').absolute()), '//a/b') @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix @@ -3195,7 +3189,7 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 3d7300e1480256..98629df457460e 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -292,6 +292,7 @@ def test_constant_folding_unaryop(self): ('---x', 'UNARY_NEGATIVE', None, False, None, None), ('~~~x', 'UNARY_INVERT', None, False, None, None), ('+++x', 'CALL_INTRINSIC_1', intrinsic_positive, False, None, None), + ('~True', 'UNARY_INVERT', None, False, None, None), ] for ( diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index d4c6fe0124af18..647c32656abd6d 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,5 +1,5 @@ import os -import sys +import sysconfig import unittest try: @@ -7,10 +7,14 @@ except ImportError: raise unittest.SkipTest("requires _testinternalcapi") +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 -if sys.platform != 'linux': - raise unittest.SkipTest('Linux only') - +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 3688cc4267b6b2..479649053abc01 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -133,6 +133,22 @@ def test_platform(self): for terse in (False, True): res = platform.platform(aliased, terse) + def test__platform(self): + for src, res in [ + ('foo bar', 'foo_bar'), + ( + '1/2\\3:4;5"6(7)8(7)6"5;4:3\\2/1', + '1-2-3-4-5-6-7-8-7-6-5-4-3-2-1' + ), + ('--', ''), + ('-f', '-f'), + ('-foo----', '-foo'), + ('--foo---', '-foo'), + ('---foo--', '-foo'), + ]: + with self.subTest(src=src): + self.assertEqual(platform._platform(src), res) + def test_system(self): res = platform.system() diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 4836f38c388c05..ed4fe8a140879d 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,6 +1,6 @@ import unittest from test.support import ( - is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_wasm32, reap_children, verbose ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -8,7 +8,7 @@ # Skip these tests if termios is not available import_module('termios') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("pty is not available on this platform") import errno diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 3e7b2cd0dc9912..bce68e6cd7a57e 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -11,7 +11,6 @@ import pyclbr from unittest import TestCase, main as unittest_main from test.test_importlib import util as test_importlib_util -import warnings StaticMethodType = type(staticmethod(lambda: None)) @@ -246,9 +245,6 @@ def test_others(self): # These were once some of the longest modules. cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('pickle', ignore=('partial', 'PickleBuffer')) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property with temporary_main_spec(): cm( 'pdb', diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index d1d6f4987def0c..3b50ead00bdd31 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -33,7 +33,7 @@ assert_python_failure, spawn_python) from test.support import threading_helper from test.support import (reap_children, captured_stdout, - captured_stderr, is_emscripten, is_wasi, + captured_stderr, is_wasm32, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test.test_pydoc import pydoc_mod @@ -1303,6 +1303,11 @@ def test_apropos_with_unreadable_dir(self): @os_helper.skip_unless_working_chmod def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') + if support.is_emscripten: + # Emscripten's readdir implementation is buggy on directories + # with read permission but no execute permission. + old_umask = os.umask(0) + self.addCleanup(os.umask, old_umask) os.mkdir(pkgdir) self.addCleanup(rmtree, pkgdir) init_path = os.path.join(pkgdir, '__init__.py') @@ -2076,7 +2081,7 @@ def test_html_doc_routines_in_module(self): @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 1d56ccd71cf962..d4b4f60be980a5 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -9,12 +9,11 @@ from io import BytesIO from test import support from test.support import os_helper - +from test.support import sortdict +from unittest import mock from xml.parsers import expat from xml.parsers.expat import errors -from test.support import sortdict - class SetAttributeTest(unittest.TestCase): def setUp(self): @@ -436,6 +435,19 @@ def test7(self): "<!--abc-->", "4", "<!--def-->", "5", "</a>"], "buffered text not properly split") + def test_change_character_data_handler_in_callback(self): + # Test that xmlparse_handler_setter() properly handles + # the special case "parser.CharacterDataHandler = None". + def handler(*args): + parser.CharacterDataHandler = None + + handler_wrapper = mock.Mock(wraps=handler) + parser = expat.ParserCreate() + parser.CharacterDataHandler = handler_wrapper + parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True) + handler_wrapper.assert_called_once() + self.assertIsNone(parser.CharacterDataHandler) + # Test handling of exception from callback: class HandlerExceptionTest(unittest.TestCase): @@ -595,7 +607,7 @@ def test_unchanged_size(self): def test_disabling_buffer(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512 xml2 = b'b' * 1024 - xml3 = b'c' * 1024 + b'</a>'; + xml3 = b'c' * 1024 + b'</a>' parser = expat.ParserCreate() parser.CharacterDataHandler = self.counting_handler parser.buffer_text = 1 diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index 8359d9844623c2..8ef472eb0cffaf 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -1,14 +1,14 @@ import os -import sys -from test.support import requires, load_package_tests -from test.support.import_helper import import_module - -if sys.platform != "win32": - # On non-Windows platforms, testing pyrepl currently requires that the - # 'curses' resource be given on the regrtest command line using the -u - # option. Additionally, we need to attempt to import curses and readline. - requires("curses") - curses = import_module("curses") +from test.support import load_package_tests +import unittest + + +try: + import termios +except ImportError: + raise unittest.SkipTest("termios required") +else: + del termios def load_tests(*args): diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index edfe6ac4748f33..69d9612b70dc77 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -3,6 +3,8 @@ from unittest.mock import patch from test import support +from _pyrepl import terminfo + try: from _pyrepl.console import Event from _pyrepl import base_eventqueue @@ -172,17 +174,22 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "a")) +class EmptyTermInfo(terminfo.TermInfo): + def get(self, cap: str) -> bytes: + return b"" + + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): def setUp(self): - self.enterContext(patch("_pyrepl.curses.tigetstr", lambda x: b"")) self.file = tempfile.TemporaryFile() def tearDown(self) -> None: self.file.close() def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: - return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8") + ti = EmptyTermInfo("ansi") + return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8", ti) @unittest.skipUnless(support.MS_WINDOWS, "No Windows event queue on Unix") diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 98bae7dd703fd9..2588a5ba27d776 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -9,10 +9,10 @@ import sys import tempfile from pkgutil import ModuleInfo -from unittest import TestCase, skipUnless, skipIf +from unittest import TestCase, skipUnless, skipIf, SkipTest from unittest.mock import patch from test.support import force_not_colorized, make_clean_env, Py_DEBUG -from test.support import SHORT_TIMEOUT, STDLIB_DIR +from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -38,6 +38,10 @@ class ReplTestCase(TestCase): + def setUp(self): + if not has_subprocess_support: + raise SkipTest("test module requires subprocess") + def run_repl( self, repl_input: str | list[str], @@ -1371,6 +1375,7 @@ def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be # added later in test methods to patched os.environ. + super().setUp() patcher = patch('os.environ', new=make_clean_env()) self.addCleanup(patcher.stop) patcher.start() @@ -1757,3 +1762,14 @@ def test_showrefcount(self): output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) self.assertEqual(len(matches), 3) + + def test_detect_pip_usage_in_repl(self): + for pip_cmd in ("pip", "pip3", "python -m pip", "python3 -m pip"): + with self.subTest(pip_cmd=pip_cmd): + output, exit_code = self.run_repl([f"{pip_cmd} install sampleproject", "exit"]) + self.assertIn("SyntaxError", output) + hint = ( + "The Python package manager (pip) can only be used" + " outside of the Python REPL" + ) + self.assertIn(hint, output) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 1f655264f1c00a..9a02dff7387563 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -375,7 +375,8 @@ def funct(case: str = sys.platform) -> None: ) match case: case "emscripten": print("on the web") - case "ios" | "android": print("on the phone") + case "ios" | "android": + print("on the phone") case _: print('arms around', match.group(1)) """ ) @@ -393,7 +394,8 @@ def funct(case: str = sys.platform) -> None: {o}){z} {K}match{z} case{o}:{z} {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} - {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} """ ) @@ -402,14 +404,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 2**7 + 2**8) - self.assertEqual(reader.cxy, (0, 14)) + self.assertEqual(reader.pos, 396) + self.assertEqual(reader.cxy, (0, 15)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 13, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 14, code_to_events("async "), ) reader, _ = handle_all_events(more_events) diff --git a/Lib/test/test_pyrepl/test_terminfo.py b/Lib/test/test_pyrepl/test_terminfo.py new file mode 100644 index 00000000000000..562cf5c905bd67 --- /dev/null +++ b/Lib/test/test_pyrepl/test_terminfo.py @@ -0,0 +1,651 @@ +"""Tests comparing PyREPL's pure Python curses implementation with the standard curses module.""" + +import json +import os +import subprocess +import sys +import unittest +from test.support import requires, has_subprocess_support +from textwrap import dedent + +# Only run these tests if curses is available +requires("curses") + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + _curses = None + +from _pyrepl import terminfo + + +ABSENT_STRING = terminfo.ABSENT_STRING +CANCELLED_STRING = terminfo.CANCELLED_STRING + + +class TestCursesCompatibility(unittest.TestCase): + """Test that PyREPL's curses implementation matches the standard curses behavior. + + Python's `curses` doesn't allow calling `setupterm()` again with a different + $TERM in the same process, so we subprocess all `curses` tests to get correctly + set up terminfo.""" + + @classmethod + def setUpClass(cls): + if _curses is None: + raise unittest.SkipTest( + "`curses` capability provided to regrtest but `_curses` not importable" + ) + + if not has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + # we need to ensure there's a terminfo database on the system and that + # `infocmp` works + cls.infocmp("dumb") + + def setUp(self): + self.original_term = os.environ.get("TERM", None) + + def tearDown(self): + if self.original_term is not None: + os.environ["TERM"] = self.original_term + elif "TERM" in os.environ: + del os.environ["TERM"] + + @classmethod + def infocmp(cls, term) -> list[str]: + all_caps = [] + try: + result = subprocess.run( + ["infocmp", "-l1", term], + capture_output=True, + text=True, + check=True, + ) + except Exception: + raise unittest.SkipTest("calling `infocmp` failed on the system") + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("#"): + if "terminfo" not in line and "termcap" in line: + # PyREPL terminfo doesn't parse termcap databases + raise unittest.SkipTest( + "curses using termcap.db: no terminfo database on" + " the system" + ) + elif "=" in line: + cap_name = line.split("=")[0] + all_caps.append(cap_name) + + return all_caps + + def test_setupterm_basic(self): + """Test basic setupterm functionality.""" + # Test with explicit terminal type + test_terms = ["xterm", "xterm-256color", "vt100", "ansi"] + + for term in test_terms: + with self.subTest(term=term): + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(term)}, 1) + print(json.dumps({{"success": True}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + # Set up with PyREPL curses + try: + terminfo.TermInfo(term, fallback=False) + pyrepl_success = True + except Exception as e: + pyrepl_success = False + pyrepl_error = e + + # Both should succeed or both should fail + if std_success: + self.assertTrue( + pyrepl_success, + f"Standard curses succeeded but PyREPL failed for {term}", + ) + else: + # If standard curses failed, PyREPL might still succeed with fallback + # This is acceptable as PyREPL has hardcoded fallbacks + pass + + def test_setupterm_none(self): + """Test setupterm with None (uses TERM from environment).""" + # Test with current TERM + ncurses_code = dedent( + """ + import _curses + import json + try: + _curses.setupterm(None, 1) + print(json.dumps({"success": True})) + except Exception as e: + print(json.dumps({"success": False, "error": str(e)})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + try: + terminfo.TermInfo(None, fallback=False) + pyrepl_success = True + except Exception: + pyrepl_success = False + + # Both should have same result + if std_success: + self.assertTrue( + pyrepl_success, + "Standard curses succeeded but PyREPL failed for None", + ) + + def test_tigetstr_common_capabilities(self): + """Test tigetstr for common terminal capabilities.""" + # Test with a known terminal type + term = "xterm" + + # Get ALL capabilities from infocmp + all_caps = self.infocmp(term) + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + + ncurses_data = json.loads(result.stdout) + + ti = terminfo.TermInfo(term, fallback=False) + + # Test every single capability + for cap in all_caps: + if cap not in ncurses_data or ncurses_data[cap] == "error": + continue + + with self.subTest(capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap}: ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_tigetstr_input_types(self): + """Test tigetstr with different input types.""" + term = "xterm" + cap = "cup" + + # Test standard curses behavior with string in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with string input + try: + std_str_result = _curses.tigetstr({repr(cap)}) + std_accepts_str = True + if std_str_result is None: + std_str_val = None + elif std_str_result == -1: + std_str_val = -1 + else: + std_str_val = list(std_str_result) + except TypeError: + std_accepts_str = False + std_str_val = None + + print(json.dumps({{ + "accepts_str": std_accepts_str, + "str_result": std_str_val + }})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # PyREPL behavior with string + try: + pyrepl_str_result = ti.get(cap) + pyrepl_accepts_str = True + except TypeError: + pyrepl_accepts_str = False + + # PyREPL should also only accept strings for compatibility + with self.assertRaises(TypeError): + ti.get(cap.encode("ascii")) + + # Both should accept string input + self.assertEqual( + pyrepl_accepts_str, + ncurses_data["accepts_str"], + "PyREPL and standard curses should have same string handling", + ) + self.assertTrue( + pyrepl_accepts_str, "PyREPL should accept string input" + ) + + def test_tparm_basic(self): + """Test basic tparm functionality.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test cursor positioning (cup) + cup = ti.get("cup") + if cup and cup not in {ABSENT_STRING, CANCELLED_STRING}: + # Test various parameter combinations + test_cases = [ + (0, 0), # Top-left + (5, 10), # Arbitrary position + (23, 79), # Bottom-right of standard terminal + (999, 999), # Large values + ] + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Get cup capability + cup = _curses.tigetstr('cup') + results = {{}} + + for row, col in {repr(test_cases)}: + try: + result = _curses.tparm(cup, row, col) + results[f"{{row}},{{col}}"] = list(result) + except Exception as e: + results[f"{{row}},{{col}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for row, col in test_cases: + with self.subTest(row=row, col=col): + # Standard curses tparm from subprocess + key = f"{row},{col}" + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + # PyREPL curses tparm + pyrepl_result = terminfo.tparm(cup, row, col) + + # Results should be identical + self.assertEqual( + pyrepl_result, + std_result, + f"tparm(cup, {row}, {col}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + else: + raise unittest.SkipTest( + "test_tparm_basic() requires the `cup` capability" + ) + + def test_tparm_multiple_params(self): + """Test tparm with capabilities using multiple parameters.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test capabilities that take parameters + param_caps = { + "cub": 1, # cursor_left with count + "cuf": 1, # cursor_right with count + "cuu": 1, # cursor_up with count + "cud": 1, # cursor_down with count + "dch": 1, # delete_character with count + "ich": 1, # insert_character with count + } + + # Get all capabilities from PyREPL first + pyrepl_caps = {} + for cap in param_caps: + cap_value = ti.get(cap) + if cap_value and cap_value not in { + ABSENT_STRING, + CANCELLED_STRING, + }: + pyrepl_caps[cap] = cap_value + + if not pyrepl_caps: + self.skipTest("No parametrized capabilities found") + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + param_caps = {repr(param_caps)} + test_values = [1, 5, 10, 99] + results = {{}} + + for cap in param_caps: + cap_value = _curses.tigetstr(cap) + if cap_value and cap_value != -1: + for value in test_values: + try: + result = _curses.tparm(cap_value, value) + results[f"{{cap}},{{value}}"] = list(result) + except Exception as e: + results[f"{{cap}},{{value}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for cap, cap_value in pyrepl_caps.items(): + with self.subTest(capability=cap): + # Test with different parameter values + for value in [1, 5, 10, 99]: + key = f"{cap},{value}" + if key in ncurses_data: + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + pyrepl_result = terminfo.tparm(cap_value, value) + self.assertEqual( + pyrepl_result, + std_result, + f"tparm({cap}, {value}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + + def test_tparm_null_handling(self): + """Test tparm with None/null input.""" + term = "xterm" + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with None + try: + _curses.tparm(None) + raises_typeerror = False + except TypeError: + raises_typeerror = True + except Exception as e: + raises_typeerror = False + error_type = type(e).__name__ + + print(json.dumps({{"raises_typeerror": raises_typeerror}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # Test with None - both should raise TypeError + if ncurses_data["raises_typeerror"]: + with self.assertRaises(TypeError): + terminfo.tparm(None) + else: + # If ncurses doesn't raise TypeError, PyREPL shouldn't either + try: + terminfo.tparm(None) + except TypeError: + self.fail("PyREPL raised TypeError but ncurses did not") + + def test_special_terminals(self): + """Test with special terminal types.""" + special_terms = [ + "dumb", # Minimal terminal + "unknown", # Should fall back to defaults + "linux", # Linux console + "screen", # GNU Screen + "tmux", # tmux + ] + + # Get all string capabilities from ncurses + for term in special_terms: + with self.subTest(term=term): + all_caps = self.infocmp(term) + ncurses_code = dedent( + f""" + import _curses + import json + import sys + + try: + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + # Convert bytes to list of ints for JSON + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + except Exception as e: + print(json.dumps({{"error": str(e)}})) + """ + ) + + # Get ncurses results + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"Failed to get ncurses data for {term}: {result.stderr}" + ) + + try: + ncurses_data = json.loads(result.stdout) + except json.JSONDecodeError: + self.fail( + f"Failed to parse ncurses output for {term}: {result.stdout}" + ) + + if "error" in ncurses_data and len(ncurses_data) == 1: + # ncurses failed to setup this terminal + # PyREPL should still work with fallback + ti = terminfo.TermInfo(term, fallback=True) + continue + + ti = terminfo.TermInfo(term, fallback=False) + + # Compare all capabilities + for cap in all_caps: + if cap not in ncurses_data: + continue + + with self.subTest(term=term, capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + # Convert back to bytes + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + # Both should return the same value + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap} for {term}: " + f"ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_terminfo_fallback(self): + """Test that PyREPL falls back gracefully when terminfo is not found.""" + # Use a non-existent terminal type + fake_term = "nonexistent-terminal-type-12345" + + # Check if standard curses can setup this terminal in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(fake_term)}, 1) + print(json.dumps({{"success": True}})) + except _curses.error: + print(json.dumps({{"success": False, "error": "curses.error"}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + if ncurses_data["success"]: + # If it succeeded, skip this test as we can't test fallback + self.skipTest( + f"System unexpectedly has terminfo for '{fake_term}'" + ) + + # PyREPL should succeed with fallback + try: + ti = terminfo.TermInfo(fake_term, fallback=True) + pyrepl_ok = True + except Exception: + pyrepl_ok = False + + self.assertTrue( + pyrepl_ok, "PyREPL should fall back for unknown terminals" + ) + + # Should still be able to get basic capabilities + bel = ti.get("bel") + self.assertIsNotNone( + bel, "PyREPL should provide basic capabilities after fallback" + ) + + def test_invalid_terminal_names(self): + cases = [ + (42, TypeError), + ("", ValueError), + ("w\x00t", ValueError), + (f"..{os.sep}name", ValueError), + ] + + for term, exc in cases: + with self.subTest(term=term): + with self.assertRaises(exc): + terminfo._validate_terminal_name_or_raise(term) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index b3f7dc028fe210..ab1236768cfb3e 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -16,9 +16,13 @@ except ImportError: pass +from _pyrepl.terminfo import _TERMINAL_CAPABILITIES + +TERM_CAPABILITIES = _TERMINAL_CAPABILITIES["ansi"] + def unix_console(events, **kwargs): - console = UnixConsole() + console = UnixConsole(term="xterm") console.get_event = MagicMock(side_effect=events) console.getpending = MagicMock(return_value=Event("key", "")) @@ -50,41 +54,11 @@ def unix_console(events, **kwargs): ) -TERM_CAPABILITIES = { - "bel": b"\x07", - "civis": b"\x1b[?25l", - "clear": b"\x1b[H\x1b[2J", - "cnorm": b"\x1b[?12l\x1b[?25h", - "cub": b"\x1b[%p1%dD", - "cub1": b"\x08", - "cud": b"\x1b[%p1%dB", - "cud1": b"\n", - "cuf": b"\x1b[%p1%dC", - "cuf1": b"\x1b[C", - "cup": b"\x1b[%i%p1%d;%p2%dH", - "cuu": b"\x1b[%p1%dA", - "cuu1": b"\x1b[A", - "dch1": b"\x1b[P", - "dch": b"\x1b[%p1%dP", - "el": b"\x1b[K", - "hpa": b"\x1b[%i%p1%dG", - "ich": b"\x1b[%p1%d@", - "ich1": None, - "ind": b"\n", - "pad": None, - "ri": b"\x1bM", - "rmkx": b"\x1b[?1l\x1b>", - "smkx": b"\x1b[?1h\x1b=", -} - - @unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") -@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) @patch( - "_pyrepl.curses.tparm", + "_pyrepl.terminfo.tparm", lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), ) -@patch("_pyrepl.curses.setupterm", lambda a, b: None) @patch( "termios.tcgetattr", lambda _: [ @@ -321,7 +295,7 @@ def same_console(events): def test_getheightwidth_with_invalid_environ(self, _os_write): # gh-128636 - console = UnixConsole() + console = UnixConsole(term="xterm") with os_helper.EnvironmentVarGuard() as env: env["LINES"] = "" self.assertIsInstance(console.getheightwidth(), tuple) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index e9128ac1d9762d..993652d2e882cc 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -5,7 +5,6 @@ import locale import re import string -import sys import unittest import warnings from re import Scanner @@ -2927,33 +2926,6 @@ def test_disallow_instantiation(self): pat = re.compile("") check_disallow_instantiation(self, type(pat.scanner(""))) - def test_deprecated_modules(self): - deprecated = { - 'sre_compile': ['compile', 'error', - 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_compile_info'], - 'sre_constants': ['error', 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_NamedIntConstant'], - 'sre_parse': ['SubPattern', 'parse', - 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', - '_parse_sub'], - } - for name in deprecated: - with self.subTest(module=name): - sys.modules.pop(name, None) - with self.assertWarns(DeprecationWarning) as w: - __import__(name) - self.assertEqual(str(w.warning), - f"module {name!r} is deprecated") - self.assertEqual(w.filename, __file__) - self.assertIn(name, sys.modules) - mod = sys.modules[name] - self.assertEqual(mod.__name__, name) - self.assertEqual(mod.__package__, '') - for attr in deprecated[name]: - self.assertHasAttr(mod, attr) - del sys.modules[name] - @cpython_only def test_case_helpers(self): import _sre diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index a1c50af15f3dd2..280e2444ef7d34 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -11,7 +11,7 @@ import unittest import unittest.mock from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack -from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT +from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT, subTests from test.support.os_helper import TESTFN, unlink from typing import List @@ -279,37 +279,50 @@ def test_handling_other_message(self): expected_stdout="Some message.\n", ) - def test_handling_help_for_command(self): - """Test handling a request to display help for a command.""" + @unittest.skipIf(sys.flags.optimize >= 2, "Help not available for -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "Usage: ll | longlist"), + # a request to display a help overview + ({"help": ""}, "type help <topic>"), + # a request to display the full PDB manual + ({"help": "pdb"}, ">>> import pdb"), + ], + ) + def test_handling_help_when_available(self, help_request, expected_substring): + """Test handling help requests when help is available.""" incoming = [ - ("server", {"help": "ll"}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="Usage: ll | longlist", + expected_stdout_substring=expected_substring, ) - def test_handling_help_without_a_specific_topic(self): - """Test handling a request to display a help overview.""" + @unittest.skipIf(sys.flags.optimize < 2, "Needs -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "No help for 'll'"), + # a request to display a help overview + ({"help": ""}, "Undocumented commands"), + # a request to display the full PDB manual + ({"help": "pdb"}, "No help for 'pdb'"), + ], + ) + def test_handling_help_when_not_available(self, help_request, expected_substring): + """Test handling help requests when help is not available.""" incoming = [ - ("server", {"help": ""}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="type help <topic>", - ) - - def test_handling_help_pdb(self): - """Test handling a request to display the full PDB manual.""" - incoming = [ - ("server", {"help": "pdb"}), - ] - self.do_test( - incoming=incoming, - expected_outgoing=[], - expected_stdout_substring=">>> import pdb", + expected_stdout_substring=expected_substring, ) def test_handling_pdb_prompts(self): diff --git a/Lib/test/test_sample_profiler.py b/Lib/test/test_sample_profiler.py new file mode 100644 index 00000000000000..2c7fa1cba712c9 --- /dev/null +++ b/Lib/test/test_sample_profiler.py @@ -0,0 +1,1877 @@ +"""Tests for the sampling profiler (profile.sample).""" + +import contextlib +import io +import marshal +import os +import socket +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from profile.pstats_collector import PstatsCollector +from profile.stack_collector import ( + CollapsedStackCollector, +) + +from test.support.os_helper import unlink +from test.support import force_not_colorized_test_class, SHORT_TIMEOUT +from test.support.socket_helper import find_unused_port +from test.support import requires_subprocess + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + import _remote_debugging +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) +else: + import profile.sample + from profile.sample import SampleProfiler + + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + + +@contextlib.contextmanager +def test_subprocess(script): + # Find an unused port for socket communication + port = find_unused_port() + + # Inject socket connection code at the beginning of the script + socket_code = f''' +import socket +_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +_test_sock.connect(('localhost', {port})) +_test_sock.sendall(b"ready") +''' + + # Combine socket code with user script + full_script = socket_code + script + + # Create server socket to wait for process to be ready + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + proc = subprocess.Popen( + [sys.executable, "-c", full_script], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + client_socket = None + try: + # Wait for process to connect and send ready signal + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + if response != b"ready": + raise RuntimeError(f"Unexpected response from subprocess: {response}") + + yield proc + finally: + if client_socket is not None: + client_socket.close() + if proc.poll() is None: + proc.kill() + proc.wait() + + +def close_and_unlink(file): + file.close() + unlink(file.name) + + +class TestSampleProfilerComponents(unittest.TestCase): + """Unit tests for individual profiler components.""" + + def test_mock_frame_info_with_empty_and_unicode_values(self): + """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" + # Test with empty strings + frame = MockFrameInfo("", 0, "") + self.assertEqual(frame.filename, "") + self.assertEqual(frame.lineno, 0) + self.assertEqual(frame.funcname, "") + self.assertIn("filename=''", repr(frame)) + + # Test with unicode characters + frame = MockFrameInfo("ๆ–‡ไปถ.py", 42, "ๅ‡ฝๆ•ฐๅ") + self.assertEqual(frame.filename, "ๆ–‡ไปถ.py") + self.assertEqual(frame.funcname, "ๅ‡ฝๆ•ฐๅ") + + # Test with very long names + long_filename = "x" * 1000 + ".py" + long_funcname = "func_" + "x" * 1000 + frame = MockFrameInfo(long_filename, 999999, long_funcname) + self.assertEqual(frame.filename, long_filename) + self.assertEqual(frame.lineno, 999999) + self.assertEqual(frame.funcname, long_funcname) + + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): + """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" + # Test with zero interval + collector = PstatsCollector(sample_interval_usec=0) + self.assertEqual(collector.sample_interval_usec, 0) + + # Test with very large interval + collector = PstatsCollector(sample_interval_usec=1000000000) + self.assertEqual(collector.sample_interval_usec, 1000000000) + + # Test collecting empty frames list + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect([]) + self.assertEqual(len(collector.result), 0) + + # Test collecting frames with None thread id + test_frames = [(None, [MockFrameInfo("file.py", 10, "func")])] + collector.collect(test_frames) + # Should still process the frames + self.assertEqual(len(collector.result), 1) + + # Test collecting duplicate frames in same sample + test_frames = [ + ( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 10, "func1"), # Duplicate + ], + ) + ] + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(test_frames) + # Should count both occurrences + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + ) + + def test_pstats_collector_single_frame_stacks(self): + """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test with exactly one frame (should trigger the <= 1 condition) + single_frame = [(1, [MockFrameInfo("single.py", 10, "single_func")])] + collector.collect(single_frame) + + # Should record the single frame with inline call + self.assertEqual(len(collector.result), 1) + single_key = ("single.py", 10, "single_func") + self.assertIn(single_key, collector.result) + self.assertEqual(collector.result[single_key]["direct_calls"], 1) + self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) + + # Test with empty frames (should also trigger <= 1 condition) + empty_frames = [(1, [])] + collector.collect(empty_frames) + + # Should not add any new entries + self.assertEqual( + len(collector.result), 1 + ) # Still just the single frame + + # Test mixed single and multi-frame stacks + mixed_frames = [ + ( + 1, + [MockFrameInfo("single2.py", 20, "single_func2")], + ), # Single frame + ( + 2, + [ # Multi-frame stack + MockFrameInfo("multi.py", 30, "multi_func1"), + MockFrameInfo("multi.py", 40, "multi_func2"), + ], + ), + ] + collector.collect(mixed_frames) + + # Should have recorded all functions + self.assertEqual( + len(collector.result), 4 + ) # single + single2 + multi1 + multi2 + + # Verify single frame handling + single2_key = ("single2.py", 20, "single_func2") + self.assertIn(single2_key, collector.result) + self.assertEqual(collector.result[single2_key]["direct_calls"], 1) + self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) + + # Verify multi-frame handling still works + multi1_key = ("multi.py", 30, "multi_func1") + multi2_key = ("multi.py", 40, "multi_func2") + self.assertIn(multi1_key, collector.result) + self.assertIn(multi2_key, collector.result) + self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) + self.assertEqual( + collector.result[multi2_key]["cumulative_calls"], 1 + ) # Called from multi1 + + def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): + """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" + collector = CollapsedStackCollector() + + # Test with empty frames + collector.collect([]) + self.assertEqual(len(collector.call_trees), 0) + + # Test with single frame stack + test_frames = [(1, [("file.py", 10, "func")])] + collector.collect(test_frames) + self.assertEqual(len(collector.call_trees), 1) + self.assertEqual(collector.call_trees[0], [("file.py", 10, "func")]) + + # Test with very deep stack + deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] + test_frames = [(1, deep_stack)] + collector = CollapsedStackCollector() + collector.collect(test_frames) + self.assertEqual(len(collector.call_trees[0]), 100) + # Check it's properly reversed + self.assertEqual( + collector.call_trees[0][0], ("file99.py", 99, "func99") + ) + self.assertEqual(collector.call_trees[0][-1], ("file0.py", 0, "func0")) + + def test_pstats_collector_basic(self): + """Test basic PstatsCollector functionality.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test empty state + self.assertEqual(len(collector.result), 0) + self.assertEqual(len(collector.stats), 0) + + # Test collecting sample data + test_frames = [ + ( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded calls for both functions + self.assertEqual(len(collector.result), 2) + self.assertIn(("file.py", 10, "func1"), collector.result) + self.assertIn(("file.py", 20, "func2"), collector.result) + + # Top-level function should have direct call + self.assertEqual( + collector.result[("file.py", 10, "func1")]["direct_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 + ) + + # Calling function should have cumulative call but no direct calls + self.assertEqual( + collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 20, "func2")]["direct_calls"], 0 + ) + + def test_pstats_collector_create_stats(self): + """Test PstatsCollector stats creation.""" + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames = [ + ( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ] + collector.collect(test_frames) + collector.collect(test_frames) # Collect twice + + collector.create_stats() + + # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) + func1_stats = collector.stats[("file.py", 10, "func1")] + self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) + self.assertEqual(func1_stats[1], 2) # cumulative_calls + self.assertEqual( + func1_stats[2], 2.0 + ) # tt (total time - 2 samples * 1 sec) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + func2_stats = collector.stats[("file.py", 20, "func2")] + self.assertEqual( + func2_stats[0], 0 + ) # direct_calls (never top of stack) + self.assertEqual( + func2_stats[1], 2 + ) # cumulative_calls (appears in stack) + self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) + self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) + + def test_collapsed_stack_collector_basic(self): + collector = CollapsedStackCollector() + + # Test empty state + self.assertEqual(len(collector.call_trees), 0) + self.assertEqual(len(collector.function_samples), 0) + + # Test collecting sample data + test_frames = [ + (1, [("file.py", 10, "func1"), ("file.py", 20, "func2")]) + ] + collector.collect(test_frames) + + # Should store call tree (reversed) + self.assertEqual(len(collector.call_trees), 1) + expected_tree = [("file.py", 20, "func2"), ("file.py", 10, "func1")] + self.assertEqual(collector.call_trees[0], expected_tree) + + # Should count function samples + self.assertEqual( + collector.function_samples[("file.py", 10, "func1")], 1 + ) + self.assertEqual( + collector.function_samples[("file.py", 20, "func2")], 1 + ) + + def test_collapsed_stack_collector_export(self): + collapsed_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, collapsed_out) + + collector = CollapsedStackCollector() + + test_frames1 = [ + (1, [("file.py", 10, "func1"), ("file.py", 20, "func2")]) + ] + test_frames2 = [ + (1, [("file.py", 10, "func1"), ("file.py", 20, "func2")]) + ] # Same stack + test_frames3 = [(1, [("other.py", 5, "other_func")])] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + collector.export(collapsed_out.name) + # Check file contents + with open(collapsed_out.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertEqual(len(lines), 2) # Two unique stacks + + # Check collapsed format: file:func:line;file:func:line count + stack1_expected = "file.py:func2:20;file.py:func1:10 2" + stack2_expected = "other.py:other_func:5 1" + + self.assertIn(stack1_expected, lines) + self.assertIn(stack2_expected, lines) + + def test_pstats_collector_export(self): + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames1 = [ + ( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ] + test_frames2 = [ + ( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ] # Same stack + test_frames3 = [(1, [MockFrameInfo("other.py", 5, "other_func")])] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + pstats_out = tempfile.NamedTemporaryFile( + suffix=".pstats", delete=False + ) + self.addCleanup(close_and_unlink, pstats_out) + collector.export(pstats_out.name) + + # Check file can be loaded with marshal + with open(pstats_out.name, "rb") as f: + stats_data = marshal.load(f) + + # Should be a dictionary with the sampled marker + self.assertIsInstance(stats_data, dict) + self.assertIn(("__sampled__",), stats_data) + self.assertTrue(stats_data[("__sampled__",)]) + + # Should have function data + function_entries = [ + k for k in stats_data.keys() if k != ("__sampled__",) + ] + self.assertGreater(len(function_entries), 0) + + # Check specific function stats format: (cc, nc, tt, ct, callers) + func1_key = ("file.py", 10, "func1") + func2_key = ("file.py", 20, "func2") + other_key = ("other.py", 5, "other_func") + + self.assertIn(func1_key, stats_data) + self.assertIn(func2_key, stats_data) + self.assertIn(other_key, stats_data) + + # Check func1 stats (should have 2 samples) + func1_stats = stats_data[func1_key] + self.assertEqual(func1_stats[0], 2) # total_calls + self.assertEqual(func1_stats[1], 2) # nc (non-recursive calls) + self.assertEqual(func1_stats[2], 2.0) # tt (total time) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + +class TestSampleProfiler(unittest.TestCase): + """Test the SampleProfiler class.""" + + def test_sample_profiler_initialization(self): + """Test SampleProfiler initialization with various parameters.""" + from profile.sample import SampleProfiler + + # Mock RemoteUnwinder to avoid permission issues + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock.MagicMock() + + # Test basic initialization + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) + self.assertEqual(profiler.pid, 12345) + self.assertEqual(profiler.sample_interval_usec, 1000) + self.assertEqual(profiler.all_threads, False) + + # Test with all_threads=True + profiler = SampleProfiler( + pid=54321, sample_interval_usec=5000, all_threads=True + ) + self.assertEqual(profiler.pid, 54321) + self.assertEqual(profiler.sample_interval_usec, 5000) + self.assertEqual(profiler.all_threads, True) + + def test_sample_profiler_sample_method_timing(self): + """Test that the sample method respects duration and handles timing correctly.""" + from profile.sample import SampleProfiler + + # Mock the unwinder to avoid needing a real process + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=100000, all_threads=False + ) # 100ms interval + + # Mock collector + mock_collector = mock.MagicMock() + + # Mock time to control the sampling loop + start_time = 1000.0 + times = [ + start_time + i * 0.1 for i in range(12) + ] # 0, 0.1, 0.2, ..., 1.1 seconds + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=1) + + result = output.getvalue() + + # Should have captured approximately 10 samples (1 second / 0.1 second interval) + self.assertIn("Captured", result) + self.assertIn("samples", result) + + # Verify collector was called multiple times + self.assertGreaterEqual(mock_collector.collect.call_count, 5) + self.assertLessEqual(mock_collector.collect.call_count, 11) + + def test_sample_profiler_error_handling(self): + """Test that the sample method handles errors gracefully.""" + from profile.sample import SampleProfiler + + # Mock unwinder that raises errors + mock_unwinder = mock.MagicMock() + error_sequence = [ + RuntimeError("Process died"), + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ], + UnicodeDecodeError("utf-8", b"", 0, 1, "invalid"), + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", + lineno=20, + funcname="test_func2", + ) + ], + ) + ], + OSError("Permission denied"), + ] + mock_unwinder.get_stack_trace.side_effect = error_sequence + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + mock_collector = mock.MagicMock() + + # Control timing to run exactly 5 samples + times = [0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06] + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.05) + + result = output.getvalue() + + # Should report error rate + self.assertIn("Error rate:", result) + self.assertIn("%", result) + + # Collector should have been called only for successful samples (should be > 0) + self.assertGreater(mock_collector.collect.call_count, 0) + self.assertLessEqual(mock_collector.collect.call_count, 3) + + def test_sample_profiler_missed_samples_warning(self): + """Test that the profiler warns about missed samples when sampling is too slow.""" + from profile.sample import SampleProfiler + + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + # Use very short interval that we'll miss + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) # 1ms interval + + mock_collector = mock.MagicMock() + + # Simulate slow sampling where we miss many samples + times = [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + ] # Extra time points to avoid StopIteration + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.5) + + result = output.getvalue() + + # Should warn about missed samples + self.assertIn("Warning: missed", result) + self.assertIn("samples from the expected total", result) + + +@force_not_colorized_test_class +class TestPrintSampledStats(unittest.TestCase): + """Test the print_sampled_stats function.""" + + def setUp(self): + """Set up test data.""" + # Mock stats data + self.mock_stats = mock.MagicMock() + self.mock_stats.stats = { + ("file1.py", 10, "func1"): ( + 100, + 100, + 0.5, + 0.5, + {}, + ), # cc, nc, tt, ct, callers + ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), + ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), + ("file4.py", 40, "func4"): ( + 10, + 10, + 0.001, + 0.001, + {}, + ), # millisecond range + ("file5.py", 50, "func5"): ( + 5, + 5, + 0.000001, + 0.000002, + {}, + ), # microsecond range + } + + def test_print_sampled_stats_basic(self): + """Test basic print_sampled_stats functionality.""" + from profile.sample import print_sampled_stats + + # Capture output + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Check header is present + self.assertIn("Profile Stats:", result) + self.assertIn("nsamples", result) + self.assertIn("tottime", result) + self.assertIn("cumtime", result) + + # Check functions are present + self.assertIn("func1", result) + self.assertIn("func2", result) + self.assertIn("func3", result) + + def test_print_sampled_stats_sorting(self): + """Test different sorting options.""" + from profile.sample import print_sampled_stats + + # Test sort by calls + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=0, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header) + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (200 calls) + self.assertIn("func3", data_lines[0]) + + # Test sort by time + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=1, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (1.5s time) + self.assertIn("func3", data_lines[0]) + + def test_print_sampled_stats_limit(self): + """Test limiting output rows.""" + from profile.sample import print_sampled_stats + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, limit=2, sample_interval_usec=100 + ) + + result = output.getvalue() + + # Count function entries in the main stats section (not in summary) + lines = result.split("\n") + # Find where the main stats section ends (before summary) + main_section_lines = [] + for line in lines: + if "Summary of Interesting Functions:" in line: + break + main_section_lines.append(line) + + # Count function entries only in main section + func_count = sum( + 1 + for line in main_section_lines + if "func" in line and ".py" in line + ) + self.assertEqual(func_count, 2) + + def test_print_sampled_stats_time_units(self): + """Test proper time unit selection.""" + from profile.sample import print_sampled_stats + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use seconds for the header since max time is > 1s + self.assertIn("tottime (s)", result) + self.assertIn("cumtime (s)", result) + + # Test with only microsecond-range times + micro_stats = mock.MagicMock() + micro_stats.stats = { + ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(micro_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use microseconds + self.assertIn("tottime (ฮผs)", result) + self.assertIn("cumtime (ฮผs)", result) + + def test_print_sampled_stats_summary(self): + """Test summary section generation.""" + from profile.sample import print_sampled_stats + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Check summary sections are present + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn( + "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", + result, + ) + self.assertIn( + "Functions with Highest Call Frequency (Indirect Calls):", result + ) + self.assertIn( + "Functions with Highest Call Magnification (Cumulative/Direct):", + result, + ) + + def test_print_sampled_stats_no_summary(self): + """Test disabling summary output.""" + from profile.sample import print_sampled_stats + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=False, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Summary should not be present + self.assertNotIn("Summary of Interesting Functions:", result) + + def test_print_sampled_stats_empty_stats(self): + """Test with empty stats.""" + from profile.sample import print_sampled_stats + + empty_stats = mock.MagicMock() + empty_stats.stats = {} + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(empty_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should still print header + self.assertIn("Profile Stats:", result) + + def test_print_sampled_stats_sample_percentage_sorting(self): + """Test sample percentage sorting options.""" + from profile.sample import print_sampled_stats + + # Add a function with high sample percentage (more direct calls than func3's 200) + self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( + 300, # direct calls (higher than func3's 200) + 300, # cumulative calls + 1.0, # total time + 1.0, # cumulative time + {}, + ) + + # Test sort by sample percentage + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=3, sample_interval_usec=100 + ) # sample percentage + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if ".py" in l and "func" in l] + # expensive_func should be first (highest sample percentage) + self.assertIn("expensive_func", data_lines[0]) + + def test_print_sampled_stats_with_recursive_calls(self): + """Test print_sampled_stats with recursive calls where nc != cc.""" + from profile.sample import print_sampled_stats + + # Create stats with recursive calls (nc != cc) + recursive_stats = mock.MagicMock() + recursive_stats.stats = { + # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function + ("recursive.py", 10, "factorial"): ( + 5, # direct_calls + 10, # cumulative_calls (appears more times in stack due to recursion) + 0.5, + 0.6, + {}, + ), + ("normal.py", 20, "normal_func"): ( + 3, # direct_calls + 3, # cumulative_calls (same as direct for non-recursive) + 0.2, + 0.2, + {}, + ), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(recursive_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should display recursive calls as "5/10" format + self.assertIn("5/10", result) # nc/cc format for recursive calls + self.assertIn("3", result) # just nc for non-recursive calls + self.assertIn("factorial", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_zero_call_counts(self): + """Test print_sampled_stats with zero call counts to trigger division protection.""" + from profile.sample import print_sampled_stats + + # Create stats with zero call counts + zero_stats = mock.MagicMock() + zero_stats.stats = { + ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls + ("file.py", 20, "normal_func"): ( + 5, + 5, + 0.1, + 0.1, + {}, + ), # Normal function + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(zero_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should handle zero call counts gracefully + self.assertIn("zero_calls", result) + self.assertIn("zero_calls", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_sort_by_name(self): + """Test sort by function name option.""" + from profile.sample import print_sampled_stats + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=-1, sample_interval_usec=100 + ) # sort by name + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header and summary) + # Data lines start with whitespace and numbers, and contain filename:lineno(function) + data_lines = [] + for line in lines: + # Skip header lines and summary sections + if ( + line.startswith(" ") + and "(" in line + and ")" in line + and not line.startswith( + " 1." + ) # Skip summary lines that start with times + and not line.startswith( + " 0." + ) # Skip summary lines that start with times + and not "per call" in line # Skip summary lines + and not "calls" in line # Skip summary lines + and not "total time" in line # Skip summary lines + and not "cumulative time" in line + ): # Skip summary lines + data_lines.append(line) + + # Extract just the function names for comparison + func_names = [] + import re + + for line in data_lines: + # Function name is between the last ( and ), accounting for ANSI color codes + match = re.search(r"\(([^)]+)\)$", line) + if match: + func_name = match.group(1) + # Remove ANSI color codes + func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) + func_names.append(func_name) + + # Verify we extracted function names and they are sorted + self.assertGreater( + len(func_names), 0, "Should have extracted some function names" + ) + self.assertEqual( + func_names, + sorted(func_names), + f"Function names {func_names} should be sorted alphabetically", + ) + + def test_print_sampled_stats_with_zero_time_functions(self): + """Test summary sections with functions that have zero time.""" + from profile.sample import print_sampled_stats + + # Create stats with zero-time functions + zero_time_stats = mock.MagicMock() + zero_time_stats.stats = { + ("file1.py", 10, "zero_time_func"): ( + 5, + 5, + 0.0, + 0.0, + {}, + ), # Zero time + ("file2.py", 20, "normal_func"): ( + 3, + 3, + 0.1, + 0.1, + {}, + ), # Normal time + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + zero_time_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle zero-time functions gracefully in summary + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn("zero_time_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_malformed_qualified_names(self): + """Test summary generation with function names that don't contain colons.""" + from profile.sample import print_sampled_stats + + # Create stats with function names that would create malformed qualified names + malformed_stats = mock.MagicMock() + malformed_stats.stats = { + # Function name without clear module separation + ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), + ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), + ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + malformed_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle malformed names gracefully in summary aggregation + self.assertIn("Summary of Interesting Functions:", result) + # All function names should appear somewhere in the output + self.assertIn("func", result) + self.assertIn("empty_filename_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_recursive_call_stats_creation(self): + """Test create_stats with recursive call data to trigger total_rec_calls branch.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second + + # Simulate recursive function data where total_rec_calls would be set + # We need to manually manipulate the collector result to test this branch + collector.result = { + ("recursive.py", 10, "factorial"): { + "total_rec_calls": 3, # Non-zero recursive calls + "direct_calls": 5, + "cumulative_calls": 10, + }, + ("normal.py", 20, "normal_func"): { + "total_rec_calls": 0, # Zero recursive calls + "direct_calls": 2, + "cumulative_calls": 5, + }, + } + + collector.create_stats() + + # Check that recursive calls are handled differently from non-recursive + factorial_stats = collector.stats[("recursive.py", 10, "factorial")] + normal_stats = collector.stats[("normal.py", 20, "normal_func")] + + # factorial should use cumulative_calls (10) as nc + self.assertEqual( + factorial_stats[1], 10 + ) # nc should be cumulative_calls + self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls + + # normal_func should use cumulative_calls as nc + self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls + self.assertEqual(normal_stats[0], 2) # cc should be direct_calls + + +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestRecursiveFunctionProfiling(unittest.TestCase): + """Test profiling of recursive functions and complex call patterns.""" + + def test_recursive_function_call_counting(self): + """Test that recursive function calls are counted correctly.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate a recursive call pattern: fibonacci(5) calling itself + recursive_frames = [ + ( + 1, + [ # First sample: deep in recursion + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call + MockFrameInfo( + "fib.py", 10, "fibonacci" + ), # deeper recursion + MockFrameInfo("fib.py", 10, "fibonacci"), # even deeper + MockFrameInfo("main.py", 5, "main"), # main caller + ], + ), + ( + 1, + [ # Second sample: different recursion depth + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call + MockFrameInfo("main.py", 5, "main"), # main caller + ], + ), + ( + 1, + [ # Third sample: back to deeper recursion + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("fib.py", 10, "fibonacci"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + ] + + for frames in recursive_frames: + collector.collect([frames]) + + collector.create_stats() + + # Check that recursive calls are counted properly + fib_key = ("fib.py", 10, "fibonacci") + main_key = ("main.py", 5, "main") + + self.assertIn(fib_key, collector.stats) + self.assertIn(main_key, collector.stats) + + # Fibonacci should have many calls due to recursion + fib_stats = collector.stats[fib_key] + direct_calls, cumulative_calls, tt, ct, callers = fib_stats + + # Should have recorded multiple calls (9 total appearances in samples) + self.assertEqual(cumulative_calls, 9) + self.assertGreater(tt, 0) # Should have some total time + self.assertGreater(ct, 0) # Should have some cumulative time + + # Main should have fewer calls + main_stats = collector.stats[main_key] + main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] + self.assertEqual(main_direct_calls, 0) # Never directly executing + self.assertEqual(main_cumulative_calls, 3) # Appears in all 3 samples + + def test_nested_function_hierarchy(self): + """Test profiling of deeply nested function calls.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate a deep call hierarchy + deep_call_frames = [ + ( + 1, + [ + MockFrameInfo("level1.py", 10, "level1_func"), + MockFrameInfo("level2.py", 20, "level2_func"), + MockFrameInfo("level3.py", 30, "level3_func"), + MockFrameInfo("level4.py", 40, "level4_func"), + MockFrameInfo("level5.py", 50, "level5_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + ( + 1, + [ # Same hierarchy sampled again + MockFrameInfo("level1.py", 10, "level1_func"), + MockFrameInfo("level2.py", 20, "level2_func"), + MockFrameInfo("level3.py", 30, "level3_func"), + MockFrameInfo("level4.py", 40, "level4_func"), + MockFrameInfo("level5.py", 50, "level5_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + ] + + for frames in deep_call_frames: + collector.collect([frames]) + + collector.create_stats() + + # All levels should be recorded + for level in range(1, 6): + key = (f"level{level}.py", level * 10, f"level{level}_func") + self.assertIn(key, collector.stats) + + stats = collector.stats[key] + direct_calls, cumulative_calls, tt, ct, callers = stats + + # Each level should appear in stack twice (2 samples) + self.assertEqual(cumulative_calls, 2) + + # Only level1 (deepest) should have direct calls + if level == 1: + self.assertEqual(direct_calls, 2) + else: + self.assertEqual(direct_calls, 0) + + # Deeper levels should have lower cumulative time than higher levels + # (since they don't include time from functions they call) + if level == 1: # Deepest level with most time + self.assertGreater(ct, 0) + + def test_alternating_call_patterns(self): + """Test profiling with alternating call patterns.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate alternating execution paths + pattern_frames = [ + # Pattern A: path through func_a + ( + 1, + [ + MockFrameInfo("module.py", 10, "func_a"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + # Pattern B: path through func_b + ( + 1, + [ + MockFrameInfo("module.py", 20, "func_b"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + # Pattern A again + ( + 1, + [ + MockFrameInfo("module.py", 10, "func_a"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + # Pattern B again + ( + 1, + [ + MockFrameInfo("module.py", 20, "func_b"), + MockFrameInfo("module.py", 30, "shared_func"), + MockFrameInfo("main.py", 5, "main"), + ], + ), + ] + + for frames in pattern_frames: + collector.collect([frames]) + + collector.create_stats() + + # Check that both paths are recorded equally + func_a_key = ("module.py", 10, "func_a") + func_b_key = ("module.py", 20, "func_b") + shared_key = ("module.py", 30, "shared_func") + main_key = ("main.py", 5, "main") + + # func_a and func_b should each be directly executing twice + self.assertEqual(collector.stats[func_a_key][0], 2) # direct_calls + self.assertEqual(collector.stats[func_a_key][1], 2) # cumulative_calls + self.assertEqual(collector.stats[func_b_key][0], 2) # direct_calls + self.assertEqual(collector.stats[func_b_key][1], 2) # cumulative_calls + + # shared_func should appear in all samples (4 times) but never directly executing + self.assertEqual(collector.stats[shared_key][0], 0) # direct_calls + self.assertEqual(collector.stats[shared_key][1], 4) # cumulative_calls + + # main should appear in all samples but never directly executing + self.assertEqual(collector.stats[main_key][0], 0) # direct_calls + self.assertEqual(collector.stats[main_key][1], 4) # cumulative_calls + + def test_collapsed_stack_with_recursion(self): + """Test collapsed stack collector with recursive patterns.""" + collector = CollapsedStackCollector() + + # Recursive call pattern + recursive_frames = [ + ( + 1, + [ + ("factorial.py", 10, "factorial"), + ("factorial.py", 10, "factorial"), # recursive + ("factorial.py", 10, "factorial"), # deeper + ("main.py", 5, "main"), + ], + ), + ( + 1, + [ + ("factorial.py", 10, "factorial"), + ("factorial.py", 10, "factorial"), # different depth + ("main.py", 5, "main"), + ], + ), + ] + + for frames in recursive_frames: + collector.collect([frames]) + + # Should capture both call trees + self.assertEqual(len(collector.call_trees), 2) + + # First tree should be longer (deeper recursion) + tree1 = collector.call_trees[0] + tree2 = collector.call_trees[1] + + # Trees should be different lengths due to different recursion depths + self.assertNotEqual(len(tree1), len(tree2)) + + # Both should contain factorial calls + self.assertTrue(any("factorial" in str(frame) for frame in tree1)) + self.assertTrue(any("factorial" in str(frame) for frame in tree2)) + + # Function samples should count all occurrences + factorial_key = ("factorial.py", 10, "factorial") + main_key = ("main.py", 5, "main") + + # factorial appears 5 times total (3 + 2) + self.assertEqual(collector.function_samples[factorial_key], 5) + # main appears 2 times total + self.assertEqual(collector.function_samples[main_key], 2) + + +@requires_subprocess() +@skip_if_not_supported +class TestSampleProfilerIntegration(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.test_script = ''' +import time +import os + +def slow_fibonacci(n): + """Recursive fibonacci - should show up prominently in profiler.""" + if n <= 1: + return n + return slow_fibonacci(n-1) + slow_fibonacci(n-2) + +def cpu_intensive_work(): + """CPU intensive work that should show in profiler.""" + result = 0 + for i in range(10000): + result += i * i + if i % 100 == 0: + result = result % 1000000 + return result + +def medium_computation(): + """Medium complexity function.""" + result = 0 + for i in range(100): + result += i * i + return result + +def fast_loop(): + """Fast simple loop.""" + total = 0 + for i in range(50): + total += i + return total + +def nested_calls(): + """Test nested function calls.""" + def level1(): + def level2(): + return medium_computation() + return level2() + return level1() + +def main_loop(): + """Main test loop with different execution paths.""" + iteration = 0 + + while True: + iteration += 1 + + # Different execution paths - focus on CPU intensive work + if iteration % 3 == 0: + # Very CPU intensive + result = cpu_intensive_work() + elif iteration % 5 == 0: + # Expensive recursive operation + result = slow_fibonacci(12) + else: + # Medium operation + result = nested_calls() + + # No sleep - keep CPU busy + +if __name__ == "__main__": + main_loop() +''' + + def test_sampling_basic_functionality(self): + with ( + test_subprocess(self.test_script) as proc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.sample( + proc.pid, + duration_sec=2, + sample_interval_usec=1000, # 1ms + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Basic checks on output + self.assertIn("Captured", output) + self.assertIn("samples", output) + self.assertIn("Profile Stats", output) + + # Should see some of our test functions + self.assertIn("slow_fibonacci", output) + + def test_sampling_with_pstats_export(self): + pstats_out = tempfile.NamedTemporaryFile( + suffix=".pstats", delete=False + ) + self.addCleanup(close_and_unlink, pstats_out) + + with test_subprocess(self.test_script) as proc: + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.sample( + proc.pid, + duration_sec=1, + filename=pstats_out.name, + sample_interval_usec=10000, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(pstats_out.name)) + self.assertGreater(os.path.getsize(pstats_out.name), 0) + + # Try to load the stats file + with open(pstats_out.name, "rb") as f: + stats_data = marshal.load(f) + + # Should be a dictionary with the sampled marker + self.assertIsInstance(stats_data, dict) + self.assertIn(("__sampled__",), stats_data) + self.assertTrue(stats_data[("__sampled__",)]) + + # Should have some function data + function_entries = [ + k for k in stats_data.keys() if k != ("__sampled__",) + ] + self.assertGreater(len(function_entries), 0) + + def test_sampling_with_collapsed_export(self): + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.test_script) as proc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.sample( + proc.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=10000, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + # Each line should have format: stack_trace count + for line in lines: + parts = line.rsplit(" ", 1) + self.assertEqual(len(parts), 2) + + stack_trace, count_str = parts + self.assertGreater(len(stack_trace), 0) + self.assertTrue(count_str.isdigit()) + self.assertGreater(int(count_str), 0) + + # Stack trace should contain semicolon-separated entries + if ";" in stack_trace: + stack_parts = stack_trace.split(";") + for part in stack_parts: + # Each part should be file:function:line + self.assertIn(":", part) + + def test_sampling_all_threads(self): + with ( + test_subprocess(self.test_script) as proc, + # Suppress profiler output + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.sample( + proc.pid, + duration_sec=1, + all_threads=True, + sample_interval_usec=10000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Just verify that sampling completed without error + # We're not testing output format here + + +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestSampleProfilerErrorHandling(unittest.TestCase): + def test_invalid_pid(self): + with self.assertRaises((OSError, RuntimeError)): + profile.sample.sample(-1, duration_sec=1) + + def test_process_dies_during_sampling(self): + with test_subprocess("import time; time.sleep(0.5); exit()") as proc: + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profile.sample.sample( + proc.pid, + duration_sec=2, # Longer than process lifetime + sample_interval_usec=50000, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + output = captured_output.getvalue() + + self.assertIn("Error rate", output) + + def test_invalid_output_format(self): + with self.assertRaises(ValueError): + profile.sample.sample( + os.getpid(), + duration_sec=1, + output_format="invalid_format", + ) + + def test_invalid_output_format_with_mocked_profiler(self): + """Test invalid output format with proper mocking to avoid permission issues.""" + with mock.patch( + "profile.sample.SampleProfiler" + ) as mock_profiler_class: + mock_profiler = mock.MagicMock() + mock_profiler_class.return_value = mock_profiler + + with self.assertRaises(ValueError) as cm: + profile.sample.sample( + 12345, + duration_sec=1, + output_format="unknown_format", + ) + + # Should raise ValueError with the invalid format name + self.assertIn( + "Invalid output format: unknown_format", str(cm.exception) + ) + + def test_is_process_running(self): + with test_subprocess("import time; time.sleep(1000)") as proc: + try: + profiler = SampleProfiler(pid=proc.pid, sample_interval_usec=1000, all_threads=False) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + self.assertTrue(profiler._is_process_running()) + self.assertIsNotNone(profiler.unwinder.get_stack_trace()) + proc.kill() + proc.wait() + # ValueError on MacOS (yeah I know), ProcessLookupError on Linux and Windows + self.assertRaises((ValueError, ProcessLookupError), profiler.unwinder.get_stack_trace) + + # Exit the context manager to ensure the process is terminated + self.assertFalse(profiler._is_process_running()) + self.assertRaises((ValueError, ProcessLookupError), profiler.unwinder.get_stack_trace) + + @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") + def test_esrch_signal_handling(self): + with test_subprocess("import time; time.sleep(1000)") as proc: + try: + unwinder = _remote_debugging.RemoteUnwinder(proc.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + initial_trace = unwinder.get_stack_trace() + self.assertIsNotNone(initial_trace) + + proc.kill() + + # Wait for the process to die and try to get another trace + proc.wait() + + with self.assertRaises(ProcessLookupError): + unwinder.get_stack_trace() + + + +class TestSampleProfilerCLI(unittest.TestCase): + def test_cli_collapsed_format_validation(self): + """Test that CLI properly validates incompatible options with collapsed format.""" + test_cases = [ + # Test sort options are invalid with collapsed + ( + ["profile.sample", "--collapsed", "--sort-nsamples", "12345"], + "sort", + ), + ( + ["profile.sample", "--collapsed", "--sort-tottime", "12345"], + "sort", + ), + ( + [ + "profile.sample", + "--collapsed", + "--sort-cumtime", + "12345", + ], + "sort", + ), + ( + [ + "profile.sample", + "--collapsed", + "--sort-sample-pct", + "12345", + ], + "sort", + ), + ( + [ + "profile.sample", + "--collapsed", + "--sort-cumul-pct", + "12345", + ], + "sort", + ), + ( + ["profile.sample", "--collapsed", "--sort-name", "12345"], + "sort", + ), + # Test limit option is invalid with collapsed + (["profile.sample", "--collapsed", "-l", "20", "12345"], "limit"), + ( + ["profile.sample", "--collapsed", "--limit", "20", "12345"], + "limit", + ), + # Test no-summary option is invalid with collapsed + ( + ["profile.sample", "--collapsed", "--no-summary", "12345"], + "summary", + ), + ] + + for test_args, expected_error_keyword in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profile.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error code + error_msg = mock_stderr.getvalue() + self.assertIn("error:", error_msg) + self.assertIn("--pstats format", error_msg) + + def test_cli_default_collapsed_filename(self): + """Test that collapsed format gets a default filename when not specified.""" + test_args = ["profile.sample", "--collapsed", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + ): + profile.sample.main() + + # Check that filename was set to default collapsed format + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["output_format"], "collapsed") + self.assertEqual(call_args["filename"], "collapsed.12345.txt") + + def test_cli_custom_output_filenames(self): + """Test custom output filenames for both formats.""" + test_cases = [ + ( + ["profile.sample", "--pstats", "-o", "custom.pstats", "12345"], + "custom.pstats", + "pstats", + ), + ( + ["profile.sample", "--collapsed", "-o", "custom.txt", "12345"], + "custom.txt", + "collapsed", + ), + ] + + for test_args, expected_filename, expected_format in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + ): + profile.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["filename"], expected_filename) + self.assertEqual(call_args["output_format"], expected_format) + + def test_cli_missing_required_arguments(self): + """Test that CLI requires PID argument.""" + with ( + mock.patch("sys.argv", ["profile.sample"]), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profile.sample.main() + + def test_cli_mutually_exclusive_format_options(self): + """Test that pstats and collapsed options are mutually exclusive.""" + with ( + mock.patch( + "sys.argv", + ["profile.sample", "--pstats", "--collapsed", "12345"], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profile.sample.main() + + def test_argument_parsing_basic(self): + test_args = ["profile.sample", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + ): + profile.sample.main() + + mock_sample.assert_called_once_with( + 12345, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + sort=2, + show_summary=True, + output_format="pstats", + realtime_stats=False, + ) + + def test_sort_options(self): + sort_options = [ + ("--sort-nsamples", 0), + ("--sort-tottime", 1), + ("--sort-cumtime", 2), + ("--sort-sample-pct", 3), + ("--sort-cumul-pct", 4), + ("--sort-name", -1), + ] + + for option, expected_sort_value in sort_options: + test_args = ["profile.sample", option, "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profile.sample.sample") as mock_sample, + ): + profile.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual( + call_args["sort"], + expected_sort_value, + ) + mock_sample.reset_mock() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py new file mode 100644 index 00000000000000..ec0ed37ffd047b --- /dev/null +++ b/Lib/test/test_samply_profiler.py @@ -0,0 +1,244 @@ +import unittest +import subprocess +import sys +import sysconfig +import os +import pathlib +from test import support +from test.support.script_helper import ( + make_script, +) +from test.support.os_helper import temp_dir + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + +if support.check_sanitizer(address=True, memory=True, ub=True, function=True): + # gh-109580: Skip the test because it does crash randomly if Python is + # built with ASAN. + raise unittest.SkipTest("test crash randomly on ASAN/MSAN/UBSAN build") + + +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 + + +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") + + +def samply_command_works(): + try: + cmd = ["samply", "--help"] + except (subprocess.SubprocessError, OSError): + return False + + # Check that we can run a simple samply run + with temp_dir() as script_dir: + try: + output_file = script_dir + "/profile.json.gz" + cmd = ( + "samply", + "record", + "--save-only", + "--output", + output_file, + sys.executable, + "-c", + 'print("hello")', + ) + env = {**os.environ, "PYTHON_JIT": "0"} + stdout = subprocess.check_output( + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + ) + except (subprocess.SubprocessError, OSError): + return False + + if "hello" not in stdout: + return False + + return True + + +def run_samply(cwd, *args, **env_vars): + env = os.environ.copy() + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + output_file = cwd + "/profile.json.gz" + base_cmd = ( + "samply", + "record", + "--save-only", + "-o", output_file, + ) + proc = subprocess.run( + base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + if proc.returncode: + print(proc.stderr, file=sys.stderr) + raise ValueError(f"Samply failed with return code {proc.returncode}") + + import gzip + with gzip.open(output_file, mode="rt", encoding="utf-8") as f: + return f.read() + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerMixin: + def run_samply(self, script_dir, perf_mode, script): + raise NotImplementedError() + + def test_python_calls_appear_in_the_stack_if_perf_activated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply(script_dir, script) + + self.assertIn(f"py::foo:{script}", output) + self.assertIn(f"py::bar:{script}", output) + self.assertIn(f"py::baz:{script}", output) + + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply( + script_dir, script, activate_trampoline=False + ) + + self.assertNotIn(f"py::foo:{script}", output) + self.assertNotIn(f"py::bar:{script}", output) + self.assertNotIn(f"py::baz:{script}", output) + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfiler(unittest.TestCase, TestSamplyProfilerMixin): + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + + def test_pre_fork_compile(self): + code = """if 1: + import sys + import os + import sysconfig + from _testinternalcapi import ( + compile_perf_trampoline_entry, + perf_trampoline_set_persist_after_fork, + ) + + def foo_fork(): + pass + + def bar_fork(): + foo_fork() + + def foo(): + import time; time.sleep(1) + + def bar(): + foo() + + def compile_trampolines_for_all_functions(): + perf_trampoline_set_persist_after_fork(1) + for _, obj in globals().items(): + if callable(obj) and hasattr(obj, '__code__'): + compile_perf_trampoline_entry(obj.__code__) + + if __name__ == "__main__": + compile_trampolines_for_all_functions() + pid = os.fork() + if pid == 0: + print(os.getpid()) + bar_fork() + else: + bar() + """ + + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, "-Xperf", script], + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0) + self.assertNotIn("Error:", stderr) + child_pid = int(stdout.strip()) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + perf_child_file = pathlib.Path(f"/tmp/perf-{child_pid}.map") + self.assertTrue(perf_file.exists()) + self.assertTrue(perf_child_file.exists()) + + perf_file_contents = perf_file.read_text() + self.assertIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", perf_file_contents) + + child_perf_file_contents = perf_child_file.read_text() + self.assertIn(f"py::foo_fork:{script}", child_perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) + + # Pre-compiled perf-map entries of a forked process must be + # identical in both the parent and child perf-map files. + perf_file_lines = perf_file_contents.split("\n") + for line in perf_file_lines: + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: + self.assertIn(line, child_perf_file_contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index 0bb02ef11f6b4b..6fc5bb74036c5f 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -154,7 +154,7 @@ class SetComprehensionTest(unittest.TestCase): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 08c6562f2a273e..64609ab9dd9a62 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,10 +1,11 @@ +import array import unittest import dbm import shelve import pickle import os -from test.support import os_helper +from test.support import import_helper, os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -165,6 +166,239 @@ def test_default_protocol(self): with shelve.Shelf({}) as s: self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) + def test_custom_serializer_and_deserializer(self): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol): + if isinstance(obj, (bytes, bytearray, str)): + if protocol == 5: + return obj + return type(obj).__name__ + elif isinstance(obj, array.array): + return obj.tobytes() + raise TypeError(f"Unsupported type for serialization: {type(obj)}") + + def deserializer(data): + if isinstance(data, (bytes, bytearray, str)): + return data.decode("utf-8") + elif isinstance(data, array.array): + return array.array("b", data) + raise TypeError( + f"Unsupported type for deserialization: {type(data)}" + ) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.open( + self.fn, + protocol=proto, + serializer=serializer, + deserializer=deserializer + ) as s: + bar = "bar" + bytes_data = b"Hello, world!" + bytearray_data = bytearray(b"\x00\x01\x02\x03\x04") + array_data = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_data + s["bytearray_data"] = bytearray_data + s["array_data"] = array_data + + if proto == 5: + self.assertEqual(s["foo"], str(bar)) + self.assertEqual(s["bytes_data"], "Hello, world!") + self.assertEqual( + s["bytearray_data"], bytearray_data.decode() + ) + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + else: + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual( + s["array_data"], array_data.tobytes().decode() + ) + + def test_custom_incomplete_serializer_and_deserializer(self): + dbm_sqlite3 = import_helper.import_module("dbm.sqlite3") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + with self.assertRaises(dbm_sqlite3.error): + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.open(self.fn, serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertNotEqual(s["foo"], "bar") + self.assertIsNone(s["foo"]) + + def test_custom_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + data = obj.__class__.__name__ + if protocol == 5: + data = str(len(data)) + return data.encode("utf-8") + + def deserializer(data): + return data.decode("utf-8") + + def type_name_len(obj): + return f"{(len(type(obj).__name__))}" + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto), shelve.BsdDbShelf( + berkeleydb.btopen(self.fn), + protocol=proto, + serializer=serializer, + deserializer=deserializer, + ) as s: + bar = "bar" + bytes_obj = b"Hello, world!" + bytearray_obj = bytearray(b"\x00\x01\x02\x03\x04") + arr_obj = array.array("i", [1, 2, 3, 4, 5]) + + s["foo"] = bar + s["bytes_data"] = bytes_obj + s["bytearray_data"] = bytearray_obj + s["array_data"] = arr_obj + + if proto == 5: + self.assertEqual(s["foo"], type_name_len(bar)) + self.assertEqual(s["bytes_data"], type_name_len(bytes_obj)) + self.assertEqual(s["bytearray_data"], + type_name_len(bytearray_obj)) + self.assertEqual(s["array_data"], type_name_len(arr_obj)) + + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, type_name_len(bar)) + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, type_name_len(bytearray_obj)) + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, type_name_len(bytes_obj)) + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, type_name_len(arr_obj)) + else: + k, v = s.set_location(b"foo") + self.assertEqual(k, "foo") + self.assertEqual(v, "str") + + k, v = s.previous() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.previous() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.previous() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + k, v = s.next() + self.assertEqual(k, "bytearray_data") + self.assertEqual(v, "bytearray") + + k, v = s.next() + self.assertEqual(k, "bytes_data") + self.assertEqual(v, "bytes") + + k, v = s.first() + self.assertEqual(k, "array_data") + self.assertEqual(v, "array") + + self.assertEqual(s["foo"], "str") + self.assertEqual(s["bytes_data"], "bytes") + self.assertEqual(s["bytearray_data"], "bytearray") + self.assertEqual(s["array_data"], "array") + + def test_custom_incomplete_serializer_and_deserializer_bsd_db_shelf(self): + berkeleydb = import_helper.import_module("berkeleydb") + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + + def serializer(obj, protocol=None): + return type(obj).__name__.encode("utf-8") + + def deserializer(data): + pass + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertIsNone(s["foo"]) + self.assertNotEqual(s["foo"], "bar") + + def serializer(obj, protocol=None): + pass + + def deserializer(data): + return data.decode("utf-8") + + with shelve.BsdDbShelf(berkeleydb.btopen(self.fn), + serializer=serializer, + deserializer=deserializer) as s: + s["foo"] = "bar" + self.assertEqual(s["foo"], "") + self.assertNotEqual(s["foo"], "bar") + + def test_missing_custom_deserializer(self): + def serializer(obj, protocol=None): + pass + + kwargs = dict(protocol=2, writeback=False, serializer=serializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + + def test_missing_custom_serializer(self): + def deserializer(data): + pass + + kwargs = dict(protocol=2, writeback=False, deserializer=deserializer) + self.assertRaises(shelve.ShelveError, shelve.Shelf, {}, **kwargs) + self.assertRaises(shelve.ShelveError, shelve.BsdDbShelf, {}, **kwargs) + class TestShelveBase: type2test = shelve.Shelf diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 291e035625397d..3602726437d8cf 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -31,8 +31,7 @@ import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_apple, is_emscripten, is_wasi + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) from test.support import gc_collect from test.support import threading_helper, import_helper @@ -641,14 +640,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + def get_undecodable_path(self): + path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -688,14 +694,10 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f123f6ece40669..9e519537ca5ed3 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4316,19 +4316,30 @@ def test_read_write_after_close_raises_valuerror(self): self.assertRaises(ValueError, s.write, b'hello') def test_sendfile(self): + """Try to send a file using kTLS if possible.""" TEST_DATA = b"x" * 512 with open(os_helper.TESTFN, 'wb') as f: f.write(TEST_DATA) self.addCleanup(os_helper.unlink, os_helper.TESTFN) client_context, server_context, hostname = testing_context() + client_context.options |= getattr(ssl, 'OP_ENABLE_KTLS', 0) server = ThreadedEchoServer(context=server_context, chatty=False) - with server: - with client_context.wrap_socket(socket.socket(), - server_hostname=hostname) as s: - s.connect((HOST, server.port)) + # kTLS seems to work only with a connection created before + # wrapping `sock` by the SSL context in contrast to calling + # `sock.connect()` after the wrapping. + with server, socket.create_connection((HOST, server.port)) as sock: + with client_context.wrap_socket( + sock, server_hostname=hostname + ) as ssock: + if support.verbose: + ktls_used = ssock._sslobj.uses_ktls_for_send() + print( + 'kTLS is', + 'available' if ktls_used else 'unavailable', + ) with open(os_helper.TESTFN, 'rb') as file: - s.sendfile(file) - self.assertEqual(s.recv(1024), TEST_DATA) + ssock.sendfile(file) + self.assertEqual(ssock.recv(1024), TEST_DATA) def test_session(self): client_context, server_context, hostname = testing_context() diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py index abdddaf187b4fe..e1d7f6f6500aeb 100644 --- a/Lib/test/test_string/_support.py +++ b/Lib/test/test_string/_support.py @@ -2,33 +2,45 @@ class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + def assertTStringEqual(self, t, strings, interpolations): """Test template string literal equality. The *strings* argument must be a tuple of strings equal to *t.strings*. The *interpolations* argument must be a sequence of tuples which are - compared against *t.interpolations*. Each tuple consists of - (value, expression, conversion, format_spec), though the final two - items may be omitted, and are assumed to be None and '' respectively. + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. """ self.assertEqual(t.strings, strings) self.assertEqual(len(t.interpolations), len(interpolations)) for i, exp in zip(t.interpolations, interpolations, strict=True): - if len(exp) == 4: - actual = (i.value, i.expression, i.conversion, i.format_spec) - self.assertEqual(actual, exp) - continue - - if len(exp) == 3: - self.assertEqual((i.value, i.expression, i.conversion), exp) - self.assertEqual(i.format_spec, '') - continue - - self.assertEqual((i.value, i.expression), exp) - self.assertEqual(i.format_spec, '') - self.assertIsNone(i.conversion) + self.assertInterpolationEqual(i, exp) def convert(value, conversion): diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index 85fcff486d6616..1c86717155fd5a 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -1,7 +1,7 @@ import pickle import unittest from collections.abc import Iterator, Iterable -from string.templatelib import Template, Interpolation +from string.templatelib import Template, Interpolation, convert from test.test_string._support import TStringBaseCase, fstring @@ -45,6 +45,19 @@ def test_basic_creation(self): self.assertEqual(len(t.interpolations), 0) self.assertEqual(fstring(t), 'Hello,\nworld') + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + def test_creation_interleaving(self): # Should add strings on either side t = Template(Interpolation('Maria', 'name', None, '')) @@ -156,5 +169,25 @@ def test_exhausted(self): self.assertRaises(StopIteration, next, template_iter) +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Cafรฉ', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index e48a2464ee5977..9ec382afb65fe4 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,6 +1,8 @@ import contextlib import errno import importlib +import itertools +import inspect import io import logging import os @@ -17,6 +19,7 @@ import warnings from test import support +from test.support import hashlib_helper from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -784,7 +787,7 @@ def test_get_signal_name(self): def test_linked_to_musl(self): linked = support.linked_to_musl() self.assertIsNotNone(linked) - if support.is_wasi or support.is_emscripten: + if support.is_wasm32: self.assertTrue(linked) # The value is cached, so make sure it returns the same value again. self.assertIs(linked, support.linked_to_musl()) @@ -818,5 +821,238 @@ def test_linked_to_musl(self): # SuppressCrashReport +@hashlib_helper.requires_builtin_hashes() +class TestHashlibSupport(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.hashlib = import_helper.import_module("hashlib") + cls.hmac = import_helper.import_module("hmac") + + # All C extension modules must be present since blocking + # the built-in implementation while allowing OpenSSL or vice-versa + # may result in failures depending on the exposed built-in hashes. + cls._hashlib = import_helper.import_module("_hashlib") + cls._hmac = import_helper.import_module("_hmac") + cls._md5 = import_helper.import_module("_md5") + + def skip_if_fips_mode(self): + if self._hashlib.get_fips_mode(): + self.skipTest("disabled in FIPS mode") + + def skip_if_not_fips_mode(self): + if not self._hashlib.get_fips_mode(): + self.skipTest("requires FIPS mode") + + def check_context(self, disabled=True): + if disabled: + return self.assertRaises(ValueError) + return contextlib.nullcontext() + + def try_import_attribute(self, fullname, default=None): + if fullname is None: + return default + assert fullname.count('.') == 1, fullname + module_name, attribute = fullname.split('.', maxsplit=1) + try: + module = importlib.import_module(module_name) + except ImportError: + return default + try: + return getattr(module, attribute, default) + except TypeError: + return default + + def fetch_hash_function(self, name, implementation): + info = hashlib_helper.get_hash_info(name) + match implementation: + case "hashlib": + assert info.hashlib is not None, info + return getattr(self.hashlib, info.hashlib) + case "openssl": + try: + return getattr(self._hashlib, info.openssl, None) + except TypeError: + return None + fullname = info.fullname(implementation) + return self.try_import_attribute(fullname) + + def fetch_hmac_function(self, name): + fullname = hashlib_helper._EXPLICIT_HMAC_CONSTRUCTORS[name] + return self.try_import_attribute(fullname) + + def check_openssl_hash(self, name, *, disabled=True): + """Check that OpenSSL HASH interface is enabled/disabled.""" + with self.check_context(disabled): + _ = self._hashlib.new(name) + if do_hash := self.fetch_hash_function(name, "openssl"): + self.assertStartsWith(do_hash.__name__, 'openssl_') + with self.check_context(disabled): + _ = do_hash(b"") + + def check_openssl_hmac(self, name, *, disabled=True): + """Check that OpenSSL HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hashlib.hmac_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hashlib.hmac_digest(b"", b"", name) + # OpenSSL does not provide one-shot explicit HMAC functions + + def check_builtin_hash(self, name, *, disabled=True): + """Check that HACL* HASH interface is enabled/disabled.""" + if do_hash := self.fetch_hash_function(name, "builtin"): + self.assertEqual(do_hash.__name__, name) + with self.check_context(disabled): + _ = do_hash(b"") + + def check_builtin_hmac(self, name, *, disabled=True): + """Check that HACL* HMAC interface is enabled/disabled.""" + if name in hashlib_helper.NON_HMAC_DIGEST_NAMES: + # HMAC-BLAKE and HMAC-SHAKE raise a ValueError as they are not + # supported at all (they do not make any sense in practice). + with self.assertRaises(ValueError): + self._hmac.compute_digest(b"", b"", name) + else: + with self.check_context(disabled): + _ = self._hmac.compute_digest(b"", b"", name) + + with self.check_context(disabled): + _ = self._hmac.new(b"", b"", name) + + if do_hmac := self.fetch_hmac_function(name): + self.assertStartsWith(do_hmac.__name__, 'compute_') + with self.check_context(disabled): + _ = do_hmac(b"", b"") + else: + self.assertIn(name, hashlib_helper.NON_HMAC_DIGEST_NAMES) + + @support.subTests( + ('name', 'allow_openssl', 'allow_builtin'), + itertools.product( + hashlib_helper.CANONICAL_DIGEST_NAMES, + [True, False], + [True, False], + ) + ) + def test_disable_hash(self, name, allow_openssl, allow_builtin): + # In FIPS mode, the function may be available but would still need + # to raise a ValueError, so we will test the helper separately. + self.skip_if_fips_mode() + flags = dict(allow_openssl=allow_openssl, allow_builtin=allow_builtin) + is_fully_disabled = not allow_builtin and not allow_openssl + + with hashlib_helper.block_algorithm(name, **flags): + # OpenSSL's blake2s and blake2b are unknown names + # when only the OpenSSL interface is available. + if allow_openssl and not allow_builtin: + aliases = {'blake2s': 'blake2s256', 'blake2b': 'blake2b512'} + name_for_hashlib_new = aliases.get(name, name) + else: + name_for_hashlib_new = name + + with self.check_context(is_fully_disabled): + _ = self.hashlib.new(name_for_hashlib_new) + + # Since _hashlib is present, explicit blake2b/blake2s constructors + # use the built-in implementation, while others (since we are not + # in FIPS mode and since _hashlib exists) use the OpenSSL function. + with self.check_context(is_fully_disabled): + _ = getattr(self.hashlib, name)() + + self.check_openssl_hash(name, disabled=not allow_openssl) + self.check_builtin_hash(name, disabled=not allow_builtin) + + if name not in hashlib_helper.NON_HMAC_DIGEST_NAMES: + with self.check_context(is_fully_disabled): + _ = self.hmac.new(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.HMAC(b"", b"", name) + with self.check_context(is_fully_disabled): + _ = self.hmac.digest(b"", b"", name) + + self.check_openssl_hmac(name, disabled=not allow_openssl) + self.check_builtin_hmac(name, disabled=not allow_builtin) + + @hashlib_helper.block_algorithm("md5") + def test_disable_hash_md5_in_fips_mode(self): + self.skip_if_not_fips_mode() + + self.assertRaises(ValueError, self.hashlib.new, "md5") + self.assertRaises(ValueError, self._hashlib.new, "md5") + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self._hashlib.openssl_md5) + + kwargs = dict(usedforsecurity=True) + self.assertRaises(ValueError, self.hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self._hashlib.new, "md5", **kwargs) + self.assertRaises(ValueError, self.hashlib.md5, **kwargs) + self.assertRaises(ValueError, self._hashlib.openssl_md5, **kwargs) + + @hashlib_helper.block_algorithm("md5", allow_openssl=True) + def test_disable_hash_md5_in_fips_mode_allow_openssl(self): + self.skip_if_not_fips_mode() + # Allow the OpenSSL interface to be used but not the HACL* one. + # hashlib.new("md5") is dispatched to hashlib.openssl_md5() + self.assertRaises(ValueError, self.hashlib.new, "md5") + # dispatched to hashlib.openssl_md5() in FIPS mode + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, self._hashlib.HASH) + + # block_algorithm() does not mock hashlib.md5 and _hashlib.openssl_md5 + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + self.assertIs(hashlib_md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + # allow MD5 to be used in FIPS mode if usedforsecurity=False + h3 = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h3, self._hashlib.HASH) + + @hashlib_helper.block_algorithm("md5", allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_builtin(self): + self.skip_if_not_fips_mode() + # Allow the HACL* interface to be used but not the OpenSSL one. + h1 = self.hashlib.new("md5") # dispatched to _md5.md5() + self.assertNotIsInstance(h1, self._hashlib.HASH) + h2 = self.hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h2, type(h1)) + + # block_algorithm() mocks hashlib.md5 and _hashlib.openssl_md5 + self.assertHasAttr(self.hashlib.md5, "__wrapped__") + self.assertHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + hashlib_md5 = inspect.unwrap(self.hashlib.md5) + openssl_md5 = inspect.unwrap(self._hashlib.openssl_md5) + self.assertIs(hashlib_md5, openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + self.assertRaises(ValueError, self.hashlib.md5, + usedforsecurity=False) + + @hashlib_helper.block_algorithm("md5", + allow_openssl=True, + allow_builtin=True) + def test_disable_hash_md5_in_fips_mode_allow_all(self): + self.skip_if_not_fips_mode() + # hashlib.new() isn't blocked as it falls back to _md5.md5 + self.assertIsInstance(self.hashlib.new("md5"), self._md5.MD5Type) + self.assertRaises(ValueError, self._hashlib.new, "md5") + h = self._hashlib.new("md5", usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + self.assertNotHasAttr(self.hashlib.md5, "__wrapped__") + self.assertNotHasAttr(self._hashlib.openssl_md5, "__wrapped__") + + self.assertIs(self.hashlib.md5, self._hashlib.openssl_md5) + self.assertRaises(ValueError, self.hashlib.md5) + h = self.hashlib.md5(usedforsecurity=False) + self.assertIsInstance(h, self._hashlib.HASH) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 24d89b09d946ad..26c75fcbc2bd7f 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -527,6 +527,58 @@ def test_symtable_entry_repr(self): expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" self.assertEqual(repr(self.top._table), expected) + def test_lambda(self): + st = symtable.symtable("lambda x: x", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_lambda(self): + st = symtable.symtable("lambda x: lambda y=x: y", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<lambda>") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), ["y"]) + self.assertEqual(st.get_children(), []) + + def test_genexpr(self): + st = symtable.symtable("(x for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(st.get_children(), []) + + def test_nested_genexpr(self): + st = symtable.symtable("((y for y in x) for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "<genexpr>") + self.assertTrue(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) + self.assertEqual(st.get_children(), []) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index b09e524a756ca8..c52d24219410c2 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2872,6 +2872,13 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) + def test_raise_from_error_message(self): + source = """if 1: + raise AssertionError() from None + print(1,,2) + """ + self._check_error(source, "invalid syntax", lineno=3) + def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2eb8de4b29fe96..4aaef5b142931e 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -708,7 +708,7 @@ def test_sysconfigdata_json(self): ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) if sysconfig._installation_is_relocated(): - ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} for key in ignore_keys: json_config_vars.pop(key, None) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 13b55d0f0a2e73..002a1feeb85c94 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1247,6 +1247,61 @@ def __del__(self): self.assertEqual(err, b"") self.assertIn(b"all clear", out) + @support.subTests('lock_class_name', ['Lock', 'RLock']) + def test_acquire_daemon_thread_lock_in_finalization(self, lock_class_name): + # gh-123940: Py_Finalize() prevents other threads from running Python + # code (and so, releasing locks), so acquiring a locked lock can not + # succeed. + # We raise an exception rather than hang. + code = textwrap.dedent(f""" + import threading + import time + + thread_started_event = threading.Event() + + lock = threading.{lock_class_name}() + def loop(): + if {lock_class_name!r} == 'RLock': + lock.acquire() + with lock: + thread_started_event.set() + while True: + time.sleep(1) + + uncontested_lock = threading.{lock_class_name}() + + class Cycle: + def __init__(self): + self.self_ref = self + self.thr = threading.Thread( + target=loop, daemon=True) + self.thr.start() + thread_started_event.wait() + + def __del__(self): + assert self.thr.is_alive() + + # We *can* acquire an unlocked lock + uncontested_lock.acquire() + if {lock_class_name!r} == 'RLock': + uncontested_lock.acquire() + + # Acquiring a locked one fails + try: + lock.acquire() + except PythonFinalizationError: + assert self.thr.is_alive() + print('got the correct exception!') + + # Cycle holds a reference to itself, which ensures it is + # cleaned up during the GC that runs after daemon threads + # have been forced to exit during finalization. + Cycle() + """) + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(err, b"") + self.assertIn(b"got the correct exception", out) + def test_start_new_thread_failed(self): # gh-109746: if Python fails to start newly created thread # due to failure of underlying PyThread_start_new_thread() call, @@ -1987,6 +2042,23 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + class ThreadRunFail(threading.Thread): def run(self): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 74b979d009664d..11b7f419bddbe4 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4748,7 +4748,26 @@ class MyList(list): with self.assertRaises(TypeError): _suggestions._generate_suggestions(MyList(), "") + def test_no_site_package_flavour(self): + code = """import boo""" + _, _, stderr = assert_python_failure('-S', '-c', code) + self.assertIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path?"), stderr + ) + + code = """ + import sys + sys.stdlib_module_names = sys.stdlib_module_names + ("boo",) + import boo + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertNotIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path?"), stderr + ) class TestColorizedTraceback(unittest.TestCase): diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index aabae38556735b..74653c77c55de1 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -150,7 +150,6 @@ def test_raw_tstrings(self): t = tr"{path}\Documents" self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) - def test_template_concatenation(self): # Test template + template t1 = t"Hello, " @@ -161,9 +160,10 @@ def test_template_concatenation(self): # Test template + string t1 = t"Hello" - combined = t1 + ", world" - self.assertTStringEqual(combined, ("Hello, world",), ()) - self.assertEqual(fstring(combined), "Hello, world") + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" # Test template + template with interpolation name = "Python" @@ -174,9 +174,10 @@ def test_template_concatenation(self): self.assertEqual(fstring(combined), "Hello, Python") # Test string + template - t = "Hello, " + t"{name}" - self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) - self.assertEqual(fstring(t), "Hello, Python") + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" def test_nested_templates(self): # Test a template inside another template expression @@ -241,52 +242,28 @@ def test_literal_concatenation(self): self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) self.assertEqual(fstring(t), "Hello, Python") - # Test concatenation with string literal - name = "Python" - t = t"Hello, {name}" "and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with Unicode literal - name = "Python" - t = t"Hello, {name}" u"and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with f-string literal - tab = '\t' - t = t"Tab: {tab}. " f"f-tab: {tab}." - self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")]) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.") - - # Test concatenation with raw string literal - tab = '\t' - t = t"Tab: {tab}. " r"Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.") - - # Test concatenation with raw f-string literal - tab = '\t' - t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.") - + # Test disallowed mix of t-string and string/f-string (incl. bytes) what = 't' - expected_msg = 'cannot mix bytes and nonbytes literals' + expected_msg = 'cannot mix t-string literals with string or bytes literals' for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", "t'{what}-string literal' b'bytes literal'", "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", ): - with self.assertRaisesRegex(SyntaxError, expected_msg): - eval(case) + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) def test_triple_quoted(self): # Test triple-quoted t-strings diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index fc26e71ffcb67b..9b0ae709d7968d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -21,6 +21,7 @@ import unittest.mock import weakref import typing +import re c_types = import_fresh_module('types', fresh=['_types']) py_types = import_fresh_module('types', blocked=['_types']) @@ -52,6 +53,7 @@ def test_names(self): 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'FrameLocalsProxyType', 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', @@ -710,6 +712,16 @@ def call(part): """ assert_python_ok("-c", code) + def test_frame_locals_proxy_type(self): + self.assertIsInstance(types.FrameLocalsProxyType, type) + self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) + self.assertEqual(types.FrameLocalsProxyType.__module__, 'builtins') + self.assertEqual(types.FrameLocalsProxyType.__name__, 'FrameLocalsProxy') + + frame = inspect.currentframe() + self.assertIsNotNone(frame) + self.assertIsInstance(frame.f_locals, types.FrameLocalsProxyType) + class UnionTests(unittest.TestCase): @@ -1375,6 +1387,27 @@ def __hash__(self): view = self.mappingproxy(mapping) self.assertEqual(hash(view), hash(mapping)) + def test_richcompare(self): + mp1 = self.mappingproxy({'a': 1}) + mp1_2 = self.mappingproxy({'a': 1}) + mp2 = self.mappingproxy({'a': 2}) + + self.assertTrue(mp1 == mp1_2) + self.assertFalse(mp1 != mp1_2) + self.assertFalse(mp1 == mp2) + self.assertTrue(mp1 != mp2) + + msg = "not supported between instances of 'mappingproxy' and 'mappingproxy'" + + with self.assertRaisesRegex(TypeError, msg): + mp1 > mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1 < mp1_2 + with self.assertRaisesRegex(TypeError, msg): + mp2 >= mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1_2 <= mp1 + class ClassCreationTests(unittest.TestCase): @@ -2009,6 +2042,24 @@ def test_equal(self): self.assertEqual(ns1, ns2) self.assertNotEqual(ns2, types.SimpleNamespace()) + def test_richcompare_unsupported(self): + ns1 = types.SimpleNamespace(x=1) + ns2 = types.SimpleNamespace(y=2) + + msg = re.escape( + "not supported between instances of " + "'types.SimpleNamespace' and 'types.SimpleNamespace'" + ) + + with self.assertRaisesRegex(TypeError, msg): + ns1 > ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 >= ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 < ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 <= ns2 + def test_nested(self): ns1 = types.SimpleNamespace(a=1, b=2) ns2 = types.SimpleNamespace() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index ef02e8202fc829..b1615bbff383c2 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1605,7 +1605,10 @@ def func1(*args: *Ts): pass self.assertEqual(gth(func1), {'args': Unpack[Ts]}) def func2(*args: *tuple[int, str]): pass - self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + hint = gth(func2)['args'] + self.assertIsInstance(hint, types.GenericAlias) + self.assertEqual(hint.__args__[0], int) + self.assertIs(hint.__unpacked__, True) class CustomVariadic(Generic[*Ts]): pass @@ -1620,7 +1623,10 @@ def func1(*args: '*Ts'): pass {'args': Unpack[Ts]}) def func2(*args: '*tuple[int, str]'): pass - self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + hint = gth(func2)['args'] + self.assertIsInstance(hint, types.GenericAlias) + self.assertEqual(hint.__args__[0], int) + self.assertIs(hint.__unpacked__, True) class CustomVariadic(Generic[*Ts]): pass @@ -6303,31 +6309,6 @@ def foo(a: 'whatevers') -> {}: class InternalsTests(BaseTestCase): - def test_deprecation_for_no_type_params_passed_to__evaluate(self): - with self.assertWarnsRegex( - DeprecationWarning, - ( - "Failing to pass a value to the 'type_params' parameter " - "of 'typing._eval_type' is deprecated" - ) - ) as cm: - self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) - - self.assertEqual(cm.filename, __file__) - - f = ForwardRef("int") - - with self.assertWarnsRegex( - DeprecationWarning, - ( - "Failing to pass a value to the 'type_params' parameter " - "of 'typing.ForwardRef._evaluate' is deprecated" - ) - ) as cm: - self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) - - self.assertEqual(cm.filename, __file__) - def test_collect_parameters(self): typing = import_helper.import_fresh_module("typing") with self.assertWarnsRegex( @@ -7114,6 +7095,24 @@ def add_right(self, node: 'Node[T]' = None): right_hints = get_type_hints(t.add_right, globals(), locals()) self.assertEqual(right_hints['node'], Node[T]) + def test_get_type_hints_preserve_generic_alias_subclasses(self): + # https://github.com/python/cpython/issues/130870 + # A real world example of this is `collections.abc.Callable`. When parameterized, + # the result is a subclass of `types.GenericAlias`. + class MyAlias(types.GenericAlias): + pass + + class MyClass: + def __class_getitem__(cls, args): + return MyAlias(cls, args) + + # Using a forward reference is important, otherwise it works as expected. + # `y` tests that the `GenericAlias` subclass is preserved when stripping `Annotated`. + def func(x: MyClass['int'], y: MyClass[Annotated[int, ...]]): ... + + assert isinstance(get_type_hints(func)['x'], MyAlias) + assert isinstance(get_type_hints(func)['y'], MyAlias) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -7327,6 +7326,12 @@ def test_partial_evaluation(self): list[EqualToForwardRef('A')], ) + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) + class CollectionsAbcTests(BaseTestCase): diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013cfbf..91d45283eb3b1b 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -12,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -480,7 +480,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio._get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -490,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index d66cab146af246..cf10e956bf2bdc 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1920,6 +1920,22 @@ def testAssertLogsUnexpectedException(self): with self.assertLogs(): raise ZeroDivisionError("Unexpected") + def testAssertLogsWithFormatter(self): + # Check alternative formats will be respected + format = "[No.1: the larch] %(levelname)s:%(name)s:%(message)s" + formatter = logging.Formatter(format) + with self.assertNoStderr(): + with self.assertLogs() as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + with self.assertLogs(formatter=formatter) as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["[No.1: the larch] INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + def testAssertNoLogsDefault(self): with self.assertRaises(self.failureException) as cm: with self.assertNoLogs(): diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 0791675b5401ca..dc36ceeb6502e8 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class AsyncClass: diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d4db5e60af7978..0d6b05bc660b76 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -206,10 +206,6 @@ def test_tstrings(self): self.check_ast_roundtrip("t'foo'") self.check_ast_roundtrip("t'foo {bar}'") self.check_ast_roundtrip("t'foo {bar!s:.2f}'") - self.check_ast_roundtrip("t'foo {bar}' f'{bar}'") - self.check_ast_roundtrip("f'{bar}' t'foo {bar}'") - self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'") - self.check_ast_roundtrip("t'foo {bar}' u'bar'") def test_strings(self): self.check_ast_roundtrip("u'foo'") diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 1d889ae7cf458f..c30fb5e27eea8a 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1526,6 +1526,14 @@ def test_url2pathname(self): self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') self.assertEqual(fn('data:blah'), 'data:blah') self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') def test_url2pathname_require_scheme(self): sep = os.path.sep diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index f66881044e16df..b8e49440c9f7da 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -89,8 +89,8 @@ def test_env_var(self): # the UTF-8 mode if not self.posix_locale(): # PYTHONUTF8 should be ignored if -E is used - out = self.get_output('-E', '-c', code, PYTHONUTF8='1') - self.assertEqual(out, '0') + out = self.get_output('-E', '-c', code, PYTHONUTF8='0') + self.assertEqual(out, '1') # invalid mode out = self.get_output('-c', code, PYTHONUTF8='xxx', failure=True) @@ -116,7 +116,7 @@ def test_filesystemencoding(self): # PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode # and has the priority over -X utf8 and PYTHONUTF8 out = self.get_output('-X', 'utf8', '-c', code, - PYTHONUTF8='strict', + PYTHONUTF8='xxx', PYTHONLEGACYWINDOWSFSENCODING='1') self.assertEqual(out, 'mbcs/replace') diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 7ddacf07a2c789..0e1a723ce3a151 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1140,6 +1140,23 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -1214,13 +1231,25 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid def check_has_stable_libuuid_extractable_node(self): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d62f3fba2d1a94..3c18c9c2900ad7 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -21,7 +21,7 @@ from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, - is_emscripten, is_wasi, + is_wasm32, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, @@ -42,7 +42,7 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 5c3b1250ceb045..f89e94449b3031 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -555,13 +555,7 @@ def test_warn_explicit_non_ascii_filename(self): with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - filenames = ["nonascii\xe9\u20ac"] - if not support.is_emscripten: - # JavaScript does not like surrogates. - # Invalid UTF-8 leading byte 0x80 encountered when - # deserializing a UTF-8 string in wasm memory to a JS - # string! - filenames.append("surrogate\udc80") + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] for filename in filenames: try: os.fsencode(filename) diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 6c3362857fc2ba..226b1aa84bd73c 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,7 @@ from test import audiotests from test import support import io +import os import struct import sys import wave @@ -196,6 +197,14 @@ def test_read_wrong_sample_width(self): with self.assertRaisesRegex(wave.Error, 'bad sample width'): wave.open(io.BytesIO(b)) + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 38be2cd437f200..bf6d5074fdebd8 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -218,6 +218,33 @@ class ElementTreeTest(unittest.TestCase): def serialize_check(self, elem, expected): self.assertEqual(serialize(elem), expected) + def test_constructor(self): + # Test constructor behavior. + + with self.assertRaises(TypeError): + tree = ET.ElementTree("") + with self.assertRaises(TypeError): + tree = ET.ElementTree(ET.ElementTree()) + + def test_setroot(self): + # Test _setroot behavior. + + tree = ET.ElementTree() + element = ET.Element("tag") + tree._setroot(element) + self.assertEqual(tree.getroot().tag, "tag") + self.assertEqual(tree.getroot(), element) + + # Test behavior with an invalid root element + + tree = ET.ElementTree() + with self.assertRaises(TypeError): + tree._setroot("") + with self.assertRaises(TypeError): + tree._setroot(ET.ElementTree()) + with self.assertRaises(TypeError): + tree._setroot(None) + def test_interface(self): # Test element tree interface. diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 9ed0f4096a45e3..270b9d6da8e7b9 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -58,7 +58,7 @@ def test_del_attribute(self): self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) @support.skip_wasi_stack_overflow() - @unittest.skipIf(support.is_emscripten, "segfaults") + @support.skip_emscripten_stack_overflow() def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 696134023a56b9..958a586b0dc8e8 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -316,7 +316,7 @@ def test_mutability(self, alpharep): HUGE_ZIPFILE_NUM_ENTRIES = 2**13 def huge_zipfile(self): - """Create a read-only zipfile with a huge number of entries entries.""" + """Create a read-only zipfile with a huge number of entries.""" strm = io.BytesIO() zf = zipfile.ZipFile(strm, "w") for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index ada96813709aea..c033059a515db6 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3470,60 +3470,6 @@ def test_execute_zip64(self): self.assertIn(b'number in executable: 5', output) -class TestDataOffsetPrependedZip(unittest.TestCase): - """Test .data_offset on reading zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') - - def _test_data_offset(self, name): - with zipfile.ZipFile(name) as zipfp: - self.assertEqual(zipfp.data_offset, 713) - - def test_data_offset_with_exe_prepended(self): - self._test_data_offset(self.exe_zip) - - def test_data_offset_with_exe_prepended_zip64(self): - self._test_data_offset(self.exe_zip64) - -class TestDataOffsetZipWrite(unittest.TestCase): - """Test .data_offset for ZipFile opened in write mode.""" - - def setUp(self): - os.mkdir(TESTFNDIR) - self.addCleanup(rmtree, TESTFNDIR) - self.test_path = os.path.join(TESTFNDIR, 'testoffset.zip') - - def test_data_offset_write_no_prefix(self): - with io.BytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 0) - - def test_data_offset_write_with_prefix(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_append_with_bad_zip(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "a") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_write_no_tell(self): - # The initializer in ZipFile checks if tell raises AttributeError or - # OSError when creating a file in write mode when deducing the offset - # of the beginning of zip data - class NoTellBytesIO(io.BytesIO): - def tell(self): - raise OSError("Unimplemented!") - with NoTellBytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertIsNone(zipfp.data_offset) - - class EncodedMetadataTests(unittest.TestCase): file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' file_content = [ diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index f313e394f49e9b..44e87e71c8ee7b 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -58,6 +58,10 @@ def tearDownModule(): shutil.rmtree(TEMP_DIR) +class CustomError(Exception): + pass + + class TzPathUserMixin: """ Adds a setUp() and tearDown() to make TZPATH manipulations thread-safe. @@ -404,6 +408,25 @@ def test_time_fixed_offset(self): self.assertEqual(t.utcoffset(), offset.utcoffset) self.assertEqual(t.dst(), offset.dst) + def test_cache_exception(self): + class Incomparable(str): + eq_called = False + def __eq__(self, other): + self.eq_called = True + raise CustomError + __hash__ = str.__hash__ + + key = "America/Los_Angeles" + tz1 = self.klass(key) + key = Incomparable(key) + try: + tz2 = self.klass(key) + except CustomError: + self.assertTrue(key.eq_called) + else: + self.assertFalse(key.eq_called) + self.assertIs(tz2, tz1) + class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo @@ -1507,6 +1530,26 @@ def test_clear_cache_two_keys(self): self.assertIsNot(dub0, dub1) self.assertIs(tok0, tok1) + def test_clear_cache_refleak(self): + class Stringy(str): + allow_comparisons = True + def __eq__(self, other): + if not self.allow_comparisons: + raise CustomError + return super().__eq__(other) + __hash__ = str.__hash__ + + key = Stringy("America/Los_Angeles") + self.klass(key) + key.allow_comparisons = False + try: + # Note: This is try/except rather than assertRaises because + # there is no guarantee that the key is even still in the cache, + # or that the key for the cache is the original `key` object. + self.klass.clear_cache(only_keys="America/Los_Angeles") + except CustomError: + pass + class CZoneInfoCacheTest(ZoneInfoCacheTest): module = c_zoneinfo diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index feaa77f3e7f0b9..c00815e2fd4c36 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -146,20 +146,22 @@ def setUp(self): @add_key_examples def test_pickle_unpickle_cache(self, key): zi = self.klass(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIs(zi, zi_rt) + self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIsNot(zi, zi_rt) - self.assertEqual(str(zi), str(zi_rt)) + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) @add_key_examples diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index d4c28aed38ef90..6358cc78739cd9 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -62,15 +62,18 @@ TRAINED_DICT = None -SUPPORT_MULTITHREADING = False +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False C_INT_MIN = -(2**31) C_INT_MAX = (2**31) - 1 def setUpModule(): - global SUPPORT_MULTITHREADING - SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) # uncompressed size 130KB, more than a zstd block. # with a frame epilogue, 4 bytes checksum. global DAT_130K_D @@ -2670,8 +2673,12 @@ def test_compress_locking(self): input = b'a'* (16*_1K) num_threads = 8 + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + comp = ZstdCompressor() - parts = [] + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] for _ in range(num_threads): res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2680,7 +2687,7 @@ def test_compress_locking(self): expected = b''.join(parts) + rest1 comp = ZstdCompressor() - output = [] + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] def run_method(method, input_data, output_data): res = method(input_data, ZstdCompressor.FLUSH_BLOCK) if res: diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 00000000000000..7347a7a42455c2 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/threading.py b/Lib/threading.py index b6c451d1fbaabd..b3e24fe7ddaa0c 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1414,7 +1414,7 @@ def __init__(self, dummy_thread): # the related _DummyThread will be kept forever! _thread_local_info._track_dummy_thread_ref = self - def __del__(self): + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): with _active_limbo_lock: if _active.get(self._tident) is self._dummy_thread: _active.pop(self._tident, None) diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py index 4f9a8815b6184b..8dcead5e31930e 100644 --- a/Lib/tkinter/scrolledtext.py +++ b/Lib/tkinter/scrolledtext.py @@ -23,7 +23,7 @@ def __init__(self, master=None, **kw): self.vbar = Scrollbar(self.frame) self.vbar.pack(side=RIGHT, fill=Y) - kw.update({'yscrollcommand': self.vbar.set}) + kw['yscrollcommand'] = self.vbar.set Text.__init__(self, self.frame, **kw) self.pack(side=LEFT, fill=BOTH, expand=True) self.vbar['command'] = self.yview diff --git a/Lib/traceback.py b/Lib/traceback.py index a1f175dbbaa421..f0dbb6352f7760 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -287,6 +287,12 @@ class FrameSummary: of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. + - :attr:`end_lineno` The last line number of the source code for this frame. + By default, it is set to lineno and indexation starts from 1. + - :attr:`colno` The column number of the source code for this frame. + By default, it is None and indexation starts from 0. + - :attr:`end_colno` The last column number of the source code for this frame. + By default, it is None and indexation starts from 0. """ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', @@ -1100,6 +1106,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) if suggestion: self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, ModuleNotFoundError) and \ + sys.flags.no_site and \ + getattr(exc_value, "name", None) not in sys.stdlib_module_names: + self._str += (". Site initialization is disabled, did you forget to " + + "add the site-packages directory to sys.path?") elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) diff --git a/Lib/types.py b/Lib/types.py index cf0549315a7814..f96c75b46daba7 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -58,7 +58,10 @@ def _m(self): pass raise TypeError except TypeError as exc: TracebackType = type(exc.__traceback__) - FrameType = type(exc.__traceback__.tb_frame) + + _f = (lambda: sys._getframe())() + FrameType = type(_f) + FrameLocalsProxyType = type(_f.f_locals) GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) diff --git a/Lib/typing.py b/Lib/typing.py index ed1dd4fc6413a5..f1455c273d31ca 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -407,6 +407,17 @@ def inner(*args, **kwds): return decorator +def _rebuild_generic_alias(alias: GenericAlias, args: tuple[object, ...]) -> GenericAlias: + is_unpacked = alias.__unpacked__ + if _should_unflatten_callable_args(alias, args): + t = alias.__origin__[(args[:-1], args[-1])] + else: + t = alias.__origin__[args] + if is_unpacked: + t = Unpack[t] + return t + + def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: import warnings @@ -426,20 +437,14 @@ def __repr__(self): return '<sentinel>' -_sentinel = _Sentinel() - - -def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), - format=None, owner=None): +def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset(), + format=None, owner=None, parent_fwdref=None): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ - if type_params is _sentinel: - _deprecation_warning_for_no_type_params_passed("typing._eval_type") - type_params = () if isinstance(t, _lazy_annotationlib.ForwardRef): # If the forward_ref has __forward_module__ set, evaluate() infers the globals # from the module, and it will probably pick better than the globals we have here. @@ -451,28 +456,23 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( - _make_forward_ref(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg for arg in t.__args__ ) - is_unpacked = t.__unpacked__ - if _should_unflatten_callable_args(t, args): - t = t.__origin__[(args[:-1], args[-1])] - else: - t = t.__origin__[args] - if is_unpacked: - t = Unpack[t] + else: + args = t.__args__ ev_args = tuple( _eval_type( a, globalns, localns, type_params, recursive_guard=recursive_guard, format=format, owner=owner, ) - for a in t.__args__ + for a in args ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): - return GenericAlias(t.__origin__, ev_args) + return _rebuild_generic_alias(t, ev_args) if isinstance(t, Union): return functools.reduce(operator.or_, ev_args) else: @@ -936,7 +936,12 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -def _make_forward_ref(code, **kwargs): +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ @@ -1001,6 +1006,7 @@ def evaluate_forward_ref( recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, format=format, owner=owner, + parent_fwdref=forward_ref, ) @@ -1860,7 +1866,9 @@ def _allow_reckless_class_checks(depth=2): The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ - return _caller(depth) in {'abc', 'functools', None} + # gh-136047: When `_abc` module is not available, `_py_abc` is required to + # allow `_py_abc.ABCMeta` fallback. + return _caller(depth) in {'abc', '_py_abc', 'functools', None} _PROTO_ALLOWLIST = { @@ -2404,7 +2412,7 @@ def _strip_annotations(t): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: return t - return GenericAlias(t.__origin__, stripped_args) + return _rebuild_generic_alias(t, stripped_args) if isinstance(t, Union): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 94868e5bb95eb3..3d69385ea243e7 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -30,7 +30,7 @@ class _AssertLogsContext(_BaseTestCaseContext): LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" - def __init__(self, test_case, logger_name, level, no_logs): + def __init__(self, test_case, logger_name, level, no_logs, formatter=None): _BaseTestCaseContext.__init__(self, test_case) self.logger_name = logger_name if level: @@ -39,13 +39,14 @@ def __init__(self, test_case, logger_name, level, no_logs): self.level = logging.INFO self.msg = None self.no_logs = no_logs + self.formatter = formatter def __enter__(self): if isinstance(self.logger_name, logging.Logger): logger = self.logger = self.logger_name else: logger = self.logger = logging.getLogger(self.logger_name) - formatter = logging.Formatter(self.LOGGING_FORMAT) + formatter = self.formatter or logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() handler.setLevel(self.level) handler.setFormatter(formatter) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index db10de68e4ac73..eba50839cd33ae 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -849,7 +849,7 @@ def _assertNotWarns(self, expected_warning, *args, **kwargs): context = _AssertNotWarnsContext(expected_warning, self) return context.handle('_assertNotWarns', args, kwargs) - def assertLogs(self, logger=None, level=None): + def assertLogs(self, logger=None, level=None, formatter=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to INFO and *logger* defaults to the root logger. @@ -861,6 +861,8 @@ def assertLogs(self, logger=None, level=None): `records` attribute will be a list of the corresponding LogRecord objects. + Optionally supply `formatter` to control how messages are formatted. + Example:: with self.assertLogs('foo', level='INFO') as cm: @@ -871,7 +873,7 @@ def assertLogs(self, logger=None, level=None): """ # Lazy import to avoid importing logging if it is not needed. from ._log import _AssertLogsContext - return _AssertLogsContext(self, logger, level, no_logs=False) + return _AssertLogsContext(self, logger, level, no_logs=False, formatter=formatter) def assertNoLogs(self, logger=None, level=None): """ Fail unless no log messages of level *level* or higher are emitted diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 41dc5d7b35dedb..c1c373d08815c1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1654,11 +1654,11 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False): The URL authority may be resolved with gethostbyname() if *resolve_host* is set to true. """ - if require_scheme: - scheme, url = _splittype(url) - if scheme != 'file': - raise URLError("URL is missing a 'file:' scheme") - authority, url = _splithost(url) + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") if os.name == 'nt': if not _is_local_authority(authority, resolve_host): # e.g. file://server/share/file.txt diff --git a/Lib/wave.py b/Lib/wave.py index 929609fa52409d..5af745e2217ec3 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -427,6 +427,8 @@ class Wave_write: _datawritten -- the size of the audio samples actually written """ + _file = None + def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 44ab5d18624e73..dafe5b1b8a0c3f 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -527,7 +527,9 @@ class ElementTree: """ def __init__(self, element=None, file=None): - # assert element is None or iselement(element) + if element is not None and not iselement(element): + raise TypeError('expected an Element, not %s' % + type(element).__name__) self._root = element # first node if file: self.parse(file) @@ -543,7 +545,9 @@ def _setroot(self, element): with the given element. Use with care! """ - # assert iselement(element) + if not iselement(element): + raise TypeError('expected an Element, not %s' + % type(element).__name__) self._root = element def parse(self, source, parser=None): @@ -709,6 +713,8 @@ def write(self, file_or_filename, of start/end tags """ + if self._root is None: + raise TypeError('ElementTree not initialized') if not method: method = "xml" elif method not in _serialize: diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 8130c739af2fe8..3e6871157d0929 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 18caeb3e04a2b5..2969f735e8abb9 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1452,7 +1452,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._lock = threading.RLock() self._seekable = True self._writing = False - self._data_offset = None try: if mode == 'r': @@ -1463,7 +1462,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._didModify = True try: self.start_dir = self.fp.tell() - self._data_offset = self.start_dir except (AttributeError, OSError): self.fp = _Tellable(self.fp) self.start_dir = 0 @@ -1488,7 +1486,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, # even if no files are added to the archive self._didModify = True self.start_dir = self.fp.tell() - self._data_offset = self.start_dir else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1535,10 +1532,6 @@ def _RealGetContents(self): # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat - # store the offset to the beginning of data for the - # .data_offset property - self._data_offset = concat - if self.start_dir < 0: raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) @@ -1599,12 +1592,6 @@ def _RealGetContents(self): zinfo._end_offset = end_offset end_offset = zinfo.header_offset - @property - def data_offset(self): - """The offset to the start of zip data in the file or None if - unavailable.""" - return self._data_offset - def namelist(self): """Return a list of file names in the archive.""" return [data.filename for data in self.filelist] diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 6e05abc32394bc..03cc42149f9b74 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -9,9 +9,13 @@ def load_tzdata(key): resource_name = components[-1] try: - return resources.files(package_name).joinpath(resource_name).open("rb") + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): - # There are three types of exception that can be raised that all amount + # There are four types of exception that can be raised that all amount # to "we cannot find this key": # # ImportError: If package_name doesn't exist (e.g. if tzdata is not diff --git a/Makefile.pre.in b/Makefile.pre.in index 66b34b779f27cb..0c070131cda200 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -804,7 +804,7 @@ build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods .PHONY: build_emscripten -build_emscripten: build_wasm web_example +build_emscripten: build_wasm web_example web_example_pyrepl_jspi # Check that the source is clean when building out of source. .PHONY: check-clean-src @@ -1095,12 +1095,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten browser web example -WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ -web_example/python.html: $(WEBEX_DIR)/python.html - @mkdir -p web_example - @cp $< $@ +EMSCRIPTEN_DIR=$(srcdir)/Tools/wasm/emscripten +WEBEX_DIR=$(EMSCRIPTEN_DIR)/web_example/ + +ZIP_STDLIB=python$(VERSION)$(ABI_THREAD).zip +$(ZIP_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(EMSCRIPTEN_DIR)/wasm_assets.py \ + Makefile pybuilddir.txt Modules/Setup.local + $(PYTHON_FOR_BUILD) $(EMSCRIPTEN_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs +web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ @@ -1108,12 +1113,9 @@ web_example/server.py: $(WEBEX_DIR)/server.py @mkdir -p web_example @cp $< $@ -WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip -$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(WEBEX_DIR)/wasm_assets.py \ - Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ - --buildroot . --prefix $(prefix) -o $@ +web_example/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p web_example + @cp $< $@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ @@ -1124,7 +1126,35 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) +web_example: web_example/python.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) + +WEBEX2=web_example_pyrepl_jspi +WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/ + +$(WEBEX2)/python.mjs $(WEBEX2)/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + @mkdir -p $(WEBEX2) + @cp python.mjs $(WEBEX2)/python.mjs + @cp python.wasm $(WEBEX2)/python.wasm + +$(WEBEX2)/index.html: $(WEBEX2_DIR)/index.html + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p $(WEBEX2) + @cp $< $@ + +.PHONY: web_example_pyrepl_jspi +web_example_pyrepl_jspi: $(WEBEX2)/python.mjs $(WEBEX2)/index.html $(WEBEX2)/src.mjs $(WEBEX2)/$(ZIP_STDLIB) + ############################################################################ # Header files @@ -1157,7 +1187,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ - $(srcdir)/Include/lock.h \ + $(srcdir)/Include/pylock.h \ $(srcdir)/Include/longobject.h \ $(srcdir)/Include/marshal.h \ $(srcdir)/Include/memoryobject.h \ @@ -1234,7 +1264,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/listobject.h \ - $(srcdir)/Include/cpython/lock.h \ + $(srcdir)/Include/cpython/pylock.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/memoryobject.h \ @@ -2530,6 +2560,7 @@ LIBSUBDIRS= asyncio \ logging \ multiprocessing multiprocessing/dummy \ pathlib \ + profile \ pydoc_data \ re \ site-packages \ diff --git a/Misc/ACKS b/Misc/ACKS index 6ab50763feadd9..745f472474cd9d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -43,6 +43,7 @@ Ray Allen Billy G. Allie Jamiel Almeida Kevin Altis +Nazฤฑm Can Altฤฑnova Samy Lahfa Skyler Leigh Amador Joe Amenta @@ -478,6 +479,7 @@ Dean Draayer Fred L. Drake, Jr. Mehdi Drissi Derk Drukker +Weilin Du John DuBois Paul Dubois Jacques Ducasse @@ -931,6 +933,7 @@ Bob Kahn Kurt B. Kaiser Tamito Kajiyama Jan Kaliszewski +Ajay Kamdar Peter van Kampen Jan Kanis Rafe Kaplan @@ -1054,6 +1057,7 @@ Alexander Lakeev David Lam Thomas Lamb Valerie Lambert +Kliment Lamonov Peter Lamut Jean-Baptiste "Jiba" Lamy Ronan Lamy @@ -1741,6 +1745,7 @@ Joel Shprentz Yue Shuaijie Jaysinh Shukla Terrel Shumway +Richard Si Eric Siegerman Reilly Tucker Siemens Paul Sijben @@ -1950,6 +1955,7 @@ Adnan Umer Utkarsh Upadhyay Roger Upole Daniel Urban +Alexander Enrique Urieles Nieto Matthias Urlichs Michael Urman Hector Urtubia @@ -1985,6 +1991,7 @@ Olivier Vielpeau Kannan Vijayan Kurt Vile Norman Vine +Tom Viner Pauli Virtanen Frank Visser Long Vo diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 3f3fb7ec599e57..6cf3db3eb43c8b 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -394,7 +394,7 @@ Removed the ``formatter`` module, which was deprecated in Python 3.4. It is somewhat obsolete, little used, and not tested. It was originally scheduled to be removed in Python 3.6, but such removals were delayed until after Python 2.7 EOL. Existing users should copy whatever classes they use into -their code. Patch by Donghee Na and and Terry J. Reedy. +their code. Patch by Donghee Na and Terry J. Reedy. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 0b49c2a78771d2..2c8e349d3c8bfb 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -2741,7 +2741,7 @@ Fix deprecation of :data:`ssl.OP_NO_TLSv1_3` .. nonce: TMWh1i .. section: Library -:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range +:meth:`!pathlib.PureWindowsPath.is_reserved` now identifies a greater range of reserved filenames, including those with trailing spaces or colons. .. diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 5dc443bb55b617..b73bbfbfdc4819 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -253,7 +253,7 @@ Adapt the ``_elementtree`` extension module to multi-phase init .. section: Library Avoid potential unexpected ``freeaddrinfo`` call (double free) in -:mod:`socket` when when a libc ``getaddrinfo()`` implementation leaves +:mod:`socket` when a libc ``getaddrinfo()`` implementation leaves garbage in an output pointer when returning an error. Original patch by Sergey G. Brester. diff --git a/Misc/NEWS.d/3.13.0a1.rst b/Misc/NEWS.d/3.13.0a1.rst index 0a93cbcea0ffd2..0741eab4eca6a4 100644 --- a/Misc/NEWS.d/3.13.0a1.rst +++ b/Misc/NEWS.d/3.13.0a1.rst @@ -153,7 +153,7 @@ about a 10% improvement. .. section: Core and Builtins Guard ``assert(tstate->thread_id > 0)`` with ``#ifndef HAVE_PTHREAD_STUBS``. -This allows for for pydebug builds to work under WASI which (currently) +This allows for pydebug builds to work under WASI which (currently) lacks thread support. .. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst index 1b971113173e0a..8afbe1b77f7cb8 100644 --- a/Misc/NEWS.d/3.13.0a4.rst +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -1096,7 +1096,7 @@ Also changed its name and daemonic status, it can be now joined. Add :func:`os.path.isreserved`, which identifies reserved pathnames such as "NUL", "AUX" and "CON". This function is only available on Windows. -Deprecate :meth:`pathlib.PurePath.is_reserved`. +Deprecate :meth:`!pathlib.PurePath.is_reserved`. .. diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 5847dea7d5e8ee..5d03d429f9ee14 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -1051,7 +1051,7 @@ warning filtering state if the :data:`sys.flags.context_aware_warnings` flag is set to true. This makes using the context manager thread-safe in multi-threaded programs. The flag is true by default in free-threaded builds and is otherwise false. The value of the flag can be overridden by -the the :option:`-X context_aware_warnings <-X>` command-line option or by +the :option:`-X context_aware_warnings <-X>` command-line option or by the :envvar:`PYTHON_CONTEXT_AWARE_WARNINGS` environment variable. .. @@ -1756,7 +1756,7 @@ Add support for macOS multi-arch builds with the JIT enabled .. nonce: q9fvyM .. section: Core and Builtins -PyREPL now supports syntax highlighing. Contributed by ลukasz Langa. +PyREPL now supports syntax highlighting. Contributed by ลukasz Langa. .. @@ -1797,7 +1797,7 @@ non-``None`` ``closure``. Patch by Bartosz Sล‚awecki. .. nonce: Uj7lyY .. section: Core and Builtins -Fix a bug that was allowing newlines inconsitently in format specifiers for +Fix a bug that was allowing newlines inconsistently in format specifiers for single-quoted f-strings. Patch by Pablo Galindo. .. @@ -2012,7 +2012,7 @@ interpreter. .. nonce: ofI5Fl .. section: C API -Add support of nullable arguments in :c:func:`PyArg_Parse` and similar +[Reverted in :gh:`136991`] Add support of nullable arguments in :c:func:`PyArg_Parse` and similar functions. Adding ``?`` after any format unit makes ``None`` be accepted as a value. diff --git a/Misc/NEWS.d/next/Build/2025-05-16-07-46-06.gh-issue-115119.ALBgS_.rst b/Misc/NEWS.d/next/Build/2025-05-16-07-46-06.gh-issue-115119.ALBgS_.rst new file mode 100644 index 00000000000000..8c2d15a3228fcf --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-16-07-46-06.gh-issue-115119.ALBgS_.rst @@ -0,0 +1,4 @@ +Removed implicit fallback to the bundled copy of the ``libmpdec`` library. +Now this should be explicitly enabled via :option:`--with-system-libmpdec` +set to ``no`` or :option:`!--without-system-libmpdec`. Patch by Sergey +B Kirpichev. diff --git a/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst b/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst new file mode 100644 index 00000000000000..fe7f962ccbb096 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst @@ -0,0 +1,2 @@ +PyREPL no longer depends on the :mod:`curses` standard library. Contributed +by ลukasz Langa. diff --git a/Misc/NEWS.d/next/C_API/2025-04-17-12-37-27.gh-issue-132629.01ArwX.rst b/Misc/NEWS.d/next/C_API/2025-04-17-12-37-27.gh-issue-132629.01ArwX.rst new file mode 100644 index 00000000000000..38b7a0a493e39f --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-04-17-12-37-27.gh-issue-132629.01ArwX.rst @@ -0,0 +1,4 @@ +For unsigned integer formats in :c:func:`PyArg_ParseTuple`, accepting Python +integers with value that is larger than the maximal value for the C type or +less than the minimal value for the corresponding signed integer type +of the same size is now deprecated. diff --git a/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst b/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst new file mode 100644 index 00000000000000..f060f09de19bb8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst @@ -0,0 +1 @@ +Expose :c:func:`PyMutex_IsLocked` as part of the public C API. diff --git a/Misc/NEWS.d/next/C_API/2025-06-19-12-47-18.gh-issue-133157.1WA85f.rst b/Misc/NEWS.d/next/C_API/2025-06-19-12-47-18.gh-issue-133157.1WA85f.rst new file mode 100644 index 00000000000000..1b37d884e57273 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-19-12-47-18.gh-issue-133157.1WA85f.rst @@ -0,0 +1 @@ +Remove the private, undocumented macro :c:macro:`!_Py_NO_SANITIZE_UNDEFINED`. diff --git a/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst b/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst new file mode 100644 index 00000000000000..41401911a6bc0b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst @@ -0,0 +1,3 @@ +New variants for the critical section API that accept one or two +:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now +public in the non-limited C API. diff --git a/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst b/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst new file mode 100644 index 00000000000000..7852759a702804 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst @@ -0,0 +1 @@ +Fix compilation errors when compiling the internal headers with a C++ compiler. diff --git a/Misc/NEWS.d/next/C_API/2025-07-01-16-22-39.gh-issue-135075.angu3J.rst b/Misc/NEWS.d/next/C_API/2025-07-01-16-22-39.gh-issue-135075.angu3J.rst new file mode 100644 index 00000000000000..88e0fa65f45dd0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-01-16-22-39.gh-issue-135075.angu3J.rst @@ -0,0 +1,2 @@ +Make :c:func:`PyObject_SetAttr` and :c:func:`PyObject_SetAttrString` fail if +called with ``NULL`` value and an exception set. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst new file mode 100644 index 00000000000000..018c5c7880c001 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst @@ -0,0 +1 @@ +Revert support of nullable arguments in :c:func:`PyArg_Parse`. diff --git a/Misc/NEWS.d/next/C_API/2025-07-23-22-30-23.gh-issue-136759.ffB4wO.rst b/Misc/NEWS.d/next/C_API/2025-07-23-22-30-23.gh-issue-136759.ffB4wO.rst new file mode 100644 index 00000000000000..79819b4735ff17 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-23-22-30-23.gh-issue-136759.ffB4wO.rst @@ -0,0 +1 @@ +Rename ``lock.h`` to ``pylock.h`` to avoid potential include conflicts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst new file mode 100644 index 00000000000000..ced7a9c9fd3e63 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst @@ -0,0 +1 @@ +Fix off-by-one read beyond the end of a string in string search. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-22-19-10.gh-issue-133711.e91wUy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-22-19-10.gh-issue-133711.e91wUy.rst new file mode 100644 index 00000000000000..c8d3d62763dc12 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-22-19-10.gh-issue-133711.e91wUy.rst @@ -0,0 +1,2 @@ +Implement :pep:`686`: Enable :ref:`Python UTF-8 Mode <utf8-mode>` by +default. Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst new file mode 100644 index 00000000000000..dc2f1e22ba5b05 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -0,0 +1 @@ +Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst new file mode 100644 index 00000000000000..f822721690975a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst @@ -0,0 +1,2 @@ +Disable constant folding for ``~`` with a boolean argument. +This moves the deprecation warning from compile time to runtime. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst new file mode 100644 index 00000000000000..9b1f62433b45ed --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst @@ -0,0 +1,3 @@ +Fixed a bug where f-string debug expressions (using =) would incorrectly +strip out parts of strings containing escaped quotes and # characters. Patch +by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst new file mode 100644 index 00000000000000..2a1ed2944d8ddf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst @@ -0,0 +1 @@ +Make functions in :mod:`grp` thread-safe on the :term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-11-19-52.gh-issue-135422.F6yQi6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-11-19-52.gh-issue-135422.F6yQi6.rst new file mode 100644 index 00000000000000..bb0f178f91ab0f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-11-19-52.gh-issue-135422.F6yQi6.rst @@ -0,0 +1 @@ +Fix regression in :exc:`SyntaxError` messages after :gh:`134036`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-08-37-45.gh-issue-82088.TgPvLg.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-08-37-45.gh-issue-82088.TgPvLg.rst new file mode 100644 index 00000000000000..74bd182c1990f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-08-37-45.gh-issue-82088.TgPvLg.rst @@ -0,0 +1,5 @@ +Improve performance of ``PyLongObject`` conversion functions +``PyLong_AsLongAndOverflow()``, ``PyLong_AsSsize_t()``, ``PyLong_AsUnsignedLong()``, ``PyLong_AsSize_t()``, +``PyLong_AsUnsignedLongMask()``, ``PyLong_AsUnsignedLongLongMask()``, ``PyLong_AsLongLongAndOverflow()`` +for integers larger than 2**30 up to 30%. + diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst new file mode 100644 index 00000000000000..859259a9ace758 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst @@ -0,0 +1,2 @@ +Fix potential :mod:`weakref` races in an object's destructor on the :term:`free threaded <free +threading>` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-02-15-18-41.gh-issue-136203.Y934sC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-02-15-18-41.gh-issue-136203.Y934sC.rst new file mode 100644 index 00000000000000..5a622bab8b893d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-02-15-18-41.gh-issue-136203.Y934sC.rst @@ -0,0 +1,2 @@ +Improve :exc:`TypeError` error message, when richcomparing two +:class:`types.MappingProxyType` objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst new file mode 100644 index 00000000000000..a37f4a51050947 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst @@ -0,0 +1 @@ +Fix memory error handling in :c:func:`PyDict_SetDefault`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-07-17-26-06.gh-issue-91636.GyHU72.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-07-17-26-06.gh-issue-91636.GyHU72.rst new file mode 100644 index 00000000000000..09c192f9c5657e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-07-17-26-06.gh-issue-91636.GyHU72.rst @@ -0,0 +1,3 @@ +While performing garbage collection, clear weakrefs to unreachable objects +that are created during running of finalizers. If those weakrefs were are +not cleared, they could reveal unreachable objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst new file mode 100644 index 00000000000000..5d59e024389c18 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst @@ -0,0 +1,5 @@ +Reflect recent :pep:`750` change. + +Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. +Also, disallow implicit concatenation of t-string literals with string or +f-string literals. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst new file mode 100644 index 00000000000000..9930413b53c150 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst @@ -0,0 +1 @@ +``Interpolation.expression`` now has a default, the empty string. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst new file mode 100644 index 00000000000000..470f3311526e83 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst @@ -0,0 +1,3 @@ +Add support for perf trampoline on macOS, to allow profilers wit JIT map +support to read Python calls. While profiling, ``PYTHONPERFSUPPORT=1`` can +be appended to enable the trampoline. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst new file mode 100644 index 00000000000000..f28eb2ca3b71e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst @@ -0,0 +1,2 @@ +Fix issue where per-thread bytecode was not instrumented for newly created +threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst new file mode 100644 index 00000000000000..bf26c4eb0e4c74 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst @@ -0,0 +1,2 @@ +Fixed a typo that prevented printing of uncollectable objects when the +:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst new file mode 100644 index 00000000000000..af9b94ad0613d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst @@ -0,0 +1,3 @@ +Fix some issues with the perf trampolines on x86-64 and aarch64. The +trampolines were not being generated correctly for some cases, which could +lead to the perf integration not working correctly. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst new file mode 100644 index 00000000000000..dcc73267a78546 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst @@ -0,0 +1 @@ +Fix crash when initializing :mod:`datetime` concurrently. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-15-10-03-57.gh-issue-116738.oFttKl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-15-10-03-57.gh-issue-116738.oFttKl.rst new file mode 100644 index 00000000000000..07d8b45fb7fbf1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-15-10-03-57.gh-issue-116738.oFttKl.rst @@ -0,0 +1 @@ +Make functions in :mod:`pwd` thread-safe on the :term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-18-08-43-35.gh-issue-116738.i0HWtP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-18-08-43-35.gh-issue-116738.i0HWtP.rst new file mode 100644 index 00000000000000..77dca4074b742f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-18-08-43-35.gh-issue-116738.i0HWtP.rst @@ -0,0 +1,2 @@ +Make functions in :mod:`syslog` thread-safe on the :term:`free threaded +<free threading>` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst new file mode 100644 index 00000000000000..767d7b97726971 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst @@ -0,0 +1 @@ +Fix PyREPL syntax highlighting on match cases after multi-line case. Contributed by Olga Matoula. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst new file mode 100644 index 00000000000000..aff047bbef0f07 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-17-08-09.gh-issue-127598.Mx8S-y.rst @@ -0,0 +1,2 @@ +Improve :exc:`ModuleNotFoundError` by adding flavour text to the exception when the +:option:`-S` option is passed. Patch by Andrea Mattei. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-24-17-30-58.gh-issue-136870.ncx82J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-24-17-30-58.gh-issue-136870.ncx82J.rst new file mode 100644 index 00000000000000..7552dc69169b67 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-24-17-30-58.gh-issue-136870.ncx82J.rst @@ -0,0 +1 @@ +Fix data races while de-instrumenting bytecode of code objects running concurrently in threads. diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst new file mode 100644 index 00000000000000..0341b5f7f0d5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst @@ -0,0 +1 @@ +EPUB builds are fixed by excluding non-XHTML-compatible tags. diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst new file mode 100644 index 00000000000000..70f54936c80f55 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-07-01-23-00-58.gh-issue-136155.4siQQO.rst @@ -0,0 +1 @@ +We are now checking for fatal errors in EPUB builds in CI. diff --git a/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst new file mode 100644 index 00000000000000..c511c63021442b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst @@ -0,0 +1,4 @@ +:mod:`ssl` now uses ``SSL_sendfile`` internally when it is possible (see +:data:`~ssl.OP_ENABLE_KTLS`). The function sends a file more efficiently +because it performs TLS encryption in the kernel to avoid additional context +switches. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst b/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst new file mode 100644 index 00000000000000..f4f2db08f73f50 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst @@ -0,0 +1 @@ +Fix flag mask inversion when unnamed flags exist. diff --git a/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst new file mode 100644 index 00000000000000..735249b4dae224 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-16-00-01-04.gh-issue-99631.GWD4fD.rst @@ -0,0 +1,2 @@ +The :mod:`shelve` module now accepts custom serialization +and deserialization functions. diff --git a/Misc/NEWS.d/next/Library/2024-10-17-01-12-22.gh-issue-119109.u4hcvb.rst b/Misc/NEWS.d/next/Library/2024-10-17-01-12-22.gh-issue-119109.u4hcvb.rst new file mode 100644 index 00000000000000..782b2eb8d74c8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-17-01-12-22.gh-issue-119109.u4hcvb.rst @@ -0,0 +1 @@ +:func:`functools.partial` calls are now faster when keyword arguments are used. diff --git a/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst b/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst new file mode 100644 index 00000000000000..dbe783a2a99e0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst @@ -0,0 +1,4 @@ +Handle corner-case for :class:`~fractions.Fraction`'s formatting: treat +zero-padding (preceding the width field by a zero (``'0'``) character) as an +equivalent to a fill character of ``'0'`` with an alignment type of ``'='``, +just as in case of :class:`float`'s. diff --git a/Misc/NEWS.d/next/Library/2025-03-19-12-41-42.gh-issue-91349.8eTOCP.rst b/Misc/NEWS.d/next/Library/2025-03-19-12-41-42.gh-issue-91349.8eTOCP.rst new file mode 100644 index 00000000000000..0e866fa4ef6169 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-19-12-41-42.gh-issue-91349.8eTOCP.rst @@ -0,0 +1,3 @@ +Adjust default ``compressionlevel=`` to 6 (down from 9) in :mod:`gzip` and :mod:`tarfile`. +It is the default level used by most compression tools and a better +tradeoff between speed and performance. diff --git a/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst b/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst new file mode 100644 index 00000000000000..cf80c71271bbd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst @@ -0,0 +1,2 @@ +Support underscore and comma as thousands separators in the fractional part +for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst b/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst new file mode 100644 index 00000000000000..be2a30d69cab45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst @@ -0,0 +1,2 @@ +Support underscore and comma as thousands separators in the fractional part +for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2025-04-08-07-25-10.gh-issue-107583.JGfbhq.rst b/Misc/NEWS.d/next/Library/2025-04-08-07-25-10.gh-issue-107583.JGfbhq.rst new file mode 100644 index 00000000000000..4235612627341b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-08-07-25-10.gh-issue-107583.JGfbhq.rst @@ -0,0 +1,4 @@ +Fix :class:`!Flag` inversion when flag set has missing values +(:class:`!IntFlag` still flips all bits); fix negative assigned values +during flag creation (both :class:`!Flag` and :class:`!IntFlag` ignore +missing values). diff --git a/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst new file mode 100644 index 00000000000000..a4b4b6d2c23d49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.combinations` and :class:`itertools.product` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst b/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst new file mode 100644 index 00000000000000..b4a2b0336370bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst @@ -0,0 +1,2 @@ +Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use +:func:`os.path.isreserved` to detect reserved paths on Windows. diff --git a/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst b/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst new file mode 100644 index 00000000000000..1bf8ee504ef180 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst @@ -0,0 +1 @@ +:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst b/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst new file mode 100644 index 00000000000000..1affb5e2aadab7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst @@ -0,0 +1,3 @@ +Fix the "Invalid error handling" exception in +:class:`!encodings.idna.IncrementalDecoder` to correctly replace the +'errors' parameter. diff --git a/Misc/NEWS.d/next/Library/2025-06-09-10-16-55.gh-issue-121914.G6Avkq.rst b/Misc/NEWS.d/next/Library/2025-06-09-10-16-55.gh-issue-121914.G6Avkq.rst new file mode 100644 index 00000000000000..a1314a9cb7f943 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-09-10-16-55.gh-issue-121914.G6Avkq.rst @@ -0,0 +1,3 @@ +Changed the names of the symbol tables for lambda expressions and generator +expressions to "<lambda>" and "<genexpr>" respectively to avoid conflicts +with user-defined names. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-10-22-18.gh-issue-130870.JipqbO.rst b/Misc/NEWS.d/next/Library/2025-06-10-10-22-18.gh-issue-130870.JipqbO.rst new file mode 100644 index 00000000000000..64173285e08417 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-10-10-22-18.gh-issue-130870.JipqbO.rst @@ -0,0 +1,2 @@ +Preserve :class:`types.GenericAlias` subclasses in +:func:`typing.get_type_hints` diff --git a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst new file mode 100644 index 00000000000000..6f395024a9e179 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.chain` safe under :term:`free threading`. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst b/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst new file mode 100644 index 00000000000000..ad217b57b4b510 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst @@ -0,0 +1,4 @@ +Address bug where it was possible to call +:func:`xml.etree.ElementTree.ElementTree.write` on an ElementTree object with +an invalid root element. This behavior blanked the file passed to ``write`` +if it already existed. diff --git a/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst b/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst new file mode 100644 index 00000000000000..f93c9faee5863a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst @@ -0,0 +1 @@ +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could occur when the Happy Eyeballs algorithm resulted in an empty exceptions list during connection attempts. diff --git a/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst b/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst new file mode 100644 index 00000000000000..1d1e7a2298c085 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst @@ -0,0 +1,3 @@ +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when non-\ :exc:`OSError` exception is raised during connection and +socket's ``close()`` raises :exc:`!OSError`. diff --git a/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst b/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst new file mode 100644 index 00000000000000..240ea72c69fa6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-24-13-30-47.gh-issue-135853.7ejTvK.rst @@ -0,0 +1,2 @@ +Add :func:`math.fmax` and :func:`math.fmin` to get the larger and smaller of +two floating-point values. Patch by Bรฉnรฉdikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-26-17-19-36.gh-issue-105456.eR9oHB.rst b/Misc/NEWS.d/next/Library/2025-06-26-17-19-36.gh-issue-105456.eR9oHB.rst new file mode 100644 index 00000000000000..772403a240aa00 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-26-17-19-36.gh-issue-105456.eR9oHB.rst @@ -0,0 +1,2 @@ +Removed :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` +modules. diff --git a/Misc/NEWS.d/next/Library/2025-06-27-09-26-04.gh-issue-87135.33z0UW.rst b/Misc/NEWS.d/next/Library/2025-06-27-09-26-04.gh-issue-87135.33z0UW.rst new file mode 100644 index 00000000000000..4b6bc74cad87a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-27-09-26-04.gh-issue-87135.33z0UW.rst @@ -0,0 +1,3 @@ +Acquiring a :class:`threading.Lock` or :class:`threading.RLock` at interpreter +shutdown will raise :exc:`PythonFinalizationError` if Python can determine +that it would otherwise deadlock. diff --git a/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst new file mode 100644 index 00000000000000..79b85320926954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst @@ -0,0 +1,2 @@ +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` when +the payload to decode is a :class:`bytes` object. Patch by Kliment Lamonov. diff --git a/Misc/NEWS.d/next/Library/2025-06-29-15-22-13.gh-issue-90733.NiquaA.rst b/Misc/NEWS.d/next/Library/2025-06-29-15-22-13.gh-issue-90733.NiquaA.rst new file mode 100644 index 00000000000000..cba930c5860522 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-29-15-22-13.gh-issue-90733.NiquaA.rst @@ -0,0 +1,2 @@ +Improve error messages when reporting invalid parameters in +:func:`hashlib.scrypt`. Patch by Bรฉnรฉdikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst b/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst new file mode 100644 index 00000000000000..fc13eb1d9e0c99 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst @@ -0,0 +1,3 @@ +If ``zoneinfo._common.load_tzdata`` is given a package without a resource a +:exc:`zoneinfo.ZoneInfoNotFoundError` is raised rather than a :exc:`PermissionError`. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-07-02-10-48-21.gh-issue-136193.xfvras.rst b/Misc/NEWS.d/next/Library/2025-07-02-10-48-21.gh-issue-136193.xfvras.rst new file mode 100644 index 00000000000000..801115202d0c95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-02-10-48-21.gh-issue-136193.xfvras.rst @@ -0,0 +1,2 @@ +Improve :exc:`TypeError` error message, when richcomparing two +:class:`types.SimpleNamespace` objects. diff --git a/Misc/NEWS.d/next/Library/2025-07-02-18-41-45.gh-issue-133982.7qqAn6.rst b/Misc/NEWS.d/next/Library/2025-07-02-18-41-45.gh-issue-133982.7qqAn6.rst new file mode 100644 index 00000000000000..a2d0810cebeaa6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-02-18-41-45.gh-issue-133982.7qqAn6.rst @@ -0,0 +1 @@ +Update Python implementation of :class:`io.BytesIO` to be thread safe. diff --git a/Misc/NEWS.d/next/Library/2025-07-04-12-53-02.gh-issue-136156.OYlXoz.rst b/Misc/NEWS.d/next/Library/2025-07-04-12-53-02.gh-issue-136156.OYlXoz.rst new file mode 100644 index 00000000000000..95606790e991a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-04-12-53-02.gh-issue-136156.OYlXoz.rst @@ -0,0 +1,3 @@ +:func:`tempfile.TemporaryFile` no longer uses :data:`os.O_EXCL` with +:data:`os.O_TMPFILE`, so it's possible to use ``linkat()`` on the file +descriptor. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst b/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst new file mode 100644 index 00000000000000..dd5cecdf3a1209 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst @@ -0,0 +1,2 @@ +Improve support for evaluating nested forward references in +:func:`typing.evaluate_forward_ref`. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-06-59-46.gh-issue-136047.qWvycf.rst b/Misc/NEWS.d/next/Library/2025-07-05-06-59-46.gh-issue-136047.qWvycf.rst new file mode 100644 index 00000000000000..1a381860914bc3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-05-06-59-46.gh-issue-136047.qWvycf.rst @@ -0,0 +1,2 @@ +Fix issues with :mod:`typing` when the C implementation of :mod:`abc` is not +available. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst new file mode 100644 index 00000000000000..ddc2310392fe92 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst @@ -0,0 +1,2 @@ +Fix pickling failures for protocols 0 and 1 for many objects related to +subinterpreters. diff --git a/Misc/NEWS.d/next/Library/2025-07-06-10-18-48.gh-issue-136021.f-FJYT.rst b/Misc/NEWS.d/next/Library/2025-07-06-10-18-48.gh-issue-136021.f-FJYT.rst new file mode 100644 index 00000000000000..39a848c11eb8a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-06-10-18-48.gh-issue-136021.f-FJYT.rst @@ -0,0 +1,3 @@ +Make ``type_params`` parameter required in :func:`!typing._eval_type` after +a deprecation period for not providing this parameter. Also remove the +:exc:`DeprecationWarning` for the old behavior. diff --git a/Misc/NEWS.d/next/Library/2025-07-06-18-38-10.gh-issue-135953.Z29DCz.rst b/Misc/NEWS.d/next/Library/2025-07-06-18-38-10.gh-issue-135953.Z29DCz.rst new file mode 100644 index 00000000000000..7e32388ed791cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-06-18-38-10.gh-issue-135953.Z29DCz.rst @@ -0,0 +1,9 @@ +Implement a new high-frequency runtime profiler that leverages the existing +remote debugging functionality to collect detailed execution statistics +from running Python processes. This tool is exposed in the +``profile.sample`` module and enables non-intrusive observation of +production applications by attaching to already-running processes without +requiring any code modifications, restarts, or special startup flags. The +observer can perform extremely high-frequency sampling of stack traces and +interpreter state, providing detailed runtime execution analysis of live +applications. diff --git a/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst b/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst new file mode 100644 index 00000000000000..f305abb655a6f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-07-16-46-55.gh-issue-72327.wLvRuj.rst @@ -0,0 +1,2 @@ +Suggest using the system command prompt when ``pip install`` is typed into +the REPL. Patch by Tom Viner, Richard Si, and Brian Schubert. diff --git a/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst b/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst new file mode 100644 index 00000000000000..4ac04b9c0a6c5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst @@ -0,0 +1,3 @@ +Raises :exc:`AttributeError` when accessing +:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are +not available. diff --git a/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst b/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst new file mode 100644 index 00000000000000..951f57100b650c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst @@ -0,0 +1,2 @@ +Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` +when running with :option:`-OO`. diff --git a/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst b/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst new file mode 100644 index 00000000000000..7634bd3be9346d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst @@ -0,0 +1,2 @@ +Fix a bug that was causing the ``get_async_stack_trace`` function to miss +some frames in the stack trace. diff --git a/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst b/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst new file mode 100644 index 00000000000000..5a0429cae07168 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst @@ -0,0 +1,2 @@ +Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread +name. diff --git a/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst b/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst new file mode 100644 index 00000000000000..a835306868d3dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst @@ -0,0 +1,3 @@ +Add missing ``keepends`` (default ``True``) parameter to +:meth:`!codecs.StreamReaderWriter.readline` and +:meth:`!codecs.StreamReaderWriter.readlines`. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst b/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst new file mode 100644 index 00000000000000..71ec66a37ef4c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst @@ -0,0 +1 @@ +Fix :class:`wave.Wave_write` emitting an unraisable when open raises. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-10-23-44.gh-issue-136492.BVi5h0.rst b/Misc/NEWS.d/next/Library/2025-07-11-10-23-44.gh-issue-136492.BVi5h0.rst new file mode 100644 index 00000000000000..7ab5b068a7f691 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-10-23-44.gh-issue-136492.BVi5h0.rst @@ -0,0 +1 @@ +Expose :pep:`667`'s :data:`~types.FrameLocalsProxyType` in the :mod:`types` module. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst new file mode 100644 index 00000000000000..f3050ad5d5aa10 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst @@ -0,0 +1 @@ +Fix signature of :func:`threading.excepthook`. diff --git a/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst b/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst new file mode 100644 index 00000000000000..ccd5bf11f0467a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-12-18-05-37.gh-issue-136591.ujXmSN.rst @@ -0,0 +1,3 @@ +:mod:`!_hashlib`: avoid using deprecated functions +:manpage:`ERR_func_error_string` and :manpage:`EVP_MD_CTX_md` when using +OpenSSL 3.0 and later. Patch by Bรฉnรฉdikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-07-15-16-37-34.gh-issue-136669.Yexwah.rst b/Misc/NEWS.d/next/Library/2025-07-15-16-37-34.gh-issue-136669.Yexwah.rst new file mode 100644 index 00000000000000..0d93397ff35d0b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-15-16-37-34.gh-issue-136669.Yexwah.rst @@ -0,0 +1 @@ +:mod:`!_asyncio` is now statically linked for improved performance. diff --git a/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst b/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst new file mode 100644 index 00000000000000..71a991aa2c5ae6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-19-15-40-47.gh-issue-131724.LS59nA.rst @@ -0,0 +1,4 @@ +In :mod:`http.client`, a new *max_response_headers* keyword-only parameter has been +added to :class:`~http.client.HTTPConnection` and :class:`~http.client.HTTPSConnection` +constructors. This parameter sets the maximum number of allowed response headers, +helping to prevent denial-of-service attacks. diff --git a/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst b/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst new file mode 100644 index 00000000000000..96e076dfe5bd12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst @@ -0,0 +1 @@ +Enable color help by default in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2025-07-20-10-21-49.gh-issue-136787._0Rbp_.rst b/Misc/NEWS.d/next/Library/2025-07-20-10-21-49.gh-issue-136787._0Rbp_.rst new file mode 100644 index 00000000000000..c6a8088e5da81f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-20-10-21-49.gh-issue-136787._0Rbp_.rst @@ -0,0 +1,4 @@ +:mod:`hashlib`: improve exception messages when a hash algorithm is not +recognized, blocked by the current security policy or incompatible with +the desired operation (for instance, using HMAC with SHAKE). +Patch by Bรฉnรฉdikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst new file mode 100644 index 00000000000000..9a71eb8ef1ac8d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst @@ -0,0 +1 @@ +Discard URL query and fragment in :func:`urllib.request.url2pathname`. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst b/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst new file mode 100644 index 00000000000000..78ec8025fbc0fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-15-40-00.gh-issue-136914.-GNG-d.rst @@ -0,0 +1,2 @@ +Fix retrieval of :attr:`doctest.DocTest.lineno` for objects decorated with +:func:`functools.cache` or :class:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst b/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst new file mode 100644 index 00000000000000..34049183649271 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst @@ -0,0 +1 @@ +pyrepl now works in Emscripten. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst b/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst new file mode 100644 index 00000000000000..fd30fe156a1b32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst @@ -0,0 +1,3 @@ +Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in 3.14.0a7 +as it wasn't fully clear which behavior it should have in some situations so +the result was not always what a user might expect. diff --git a/Misc/NEWS.d/next/Library/2025-07-23-11-59-48.gh-issue-136980.BIJzkB.rst b/Misc/NEWS.d/next/Library/2025-07-23-11-59-48.gh-issue-136980.BIJzkB.rst new file mode 100644 index 00000000000000..a7111dd2b865ba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-23-11-59-48.gh-issue-136980.BIJzkB.rst @@ -0,0 +1 @@ +Remove unused C tracing code in bdb for event type ``c_call``, ``c_return`` and ``c_exception`` diff --git a/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst b/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst new file mode 100644 index 00000000000000..6c2246631dda9e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-25-09-21-56.gh-issue-130522.Crwq68.rst @@ -0,0 +1,2 @@ +Fix unraisable :exc:`TypeError` raised during :term:`interpreter shutdown` +in the :mod:`threading` module. diff --git a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst new file mode 100644 index 00000000000000..6ad3caf33b2201 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst @@ -0,0 +1,2 @@ +Fix support of escapable raw text mode (elements "textarea" and "title") +in :class:`html.parser.HTMLParser`. diff --git a/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst b/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst new file mode 100644 index 00000000000000..71d15ee0852ebd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst @@ -0,0 +1,3 @@ +Fix comment parsing in :class:`html.parser.HTMLParser` according to the +HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. diff --git a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst new file mode 100644 index 00000000000000..27e886abdb58e5 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst @@ -0,0 +1,20 @@ +Fix parsing start and end tags in :class:`html.parser.HTMLParser` +according to the HTML5 standard. + +* Whitespaces no longer accepted between ``</`` and the tag name. + E.g. ``</ script>`` does not end the script section. + +* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized + as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. + +* Null character (U+0000) no longer ends the tag name. + +* Attributes and slashes after the tag name in end tags are now ignored, + instead of terminating after the first ``>`` in quoted attribute value. + E.g. ``</script/foo=">"/>``. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ``<a foo=bar/ //>``. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ``<a foo==bar>`` produces attribute "foo" with value "=bar". diff --git a/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst b/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst new file mode 100644 index 00000000000000..93caed3aa3b9dd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst @@ -0,0 +1 @@ +:mod:`marshal`: fix a possible crash when deserializing :class:`slice` objects. diff --git a/Misc/NEWS.d/next/Tests/2025-05-23-09-19-52.gh-issue-134567.hwEIMb.rst b/Misc/NEWS.d/next/Tests/2025-05-23-09-19-52.gh-issue-134567.hwEIMb.rst new file mode 100644 index 00000000000000..42e4a01c0ccb4b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-05-23-09-19-52.gh-issue-134567.hwEIMb.rst @@ -0,0 +1,2 @@ +Expose log formatter to users in TestCase.assertLogs. +:func:`unittest.TestCase.assertLogs` will now optionally accept a formatter that will be used to format the strings in output if provided. diff --git a/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst b/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst new file mode 100644 index 00000000000000..6885fba30dbab0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-06-11-16-52-49.gh-issue-135401.ccMXmL.rst @@ -0,0 +1 @@ +Add a new GitHub CI job to test the :mod:`ssl` module with `AWS-LC <https://github.com/aws/aws-lc>`_ as the backing cryptography and TLS library. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst index 25599a865b7246..ebe3ab0e7d1993 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2025-06-11-12-14-06.gh-issue-135379.25ttXq.rst @@ -1,4 +1,4 @@ The cases generator no longer accepts type annotations on stack items. -Conversions to non-default types are now done explictly in bytecodes.c and +Conversions to non-default types are now done explicitly in bytecodes.c and optimizer_bytecodes.c. This will simplify code generation for top-of-stack caching and other future features. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst b/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst new file mode 100644 index 00000000000000..6a35afe15e3875 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst @@ -0,0 +1 @@ +Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 2b2e8cb3e3cacd..65a1fefe72e92e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -12,6 +12,8 @@ posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c _suggestions _suggestions.c +# needs libm and on some platforms librt +_datetime _datetimemodule.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3a38a60a152e8c..7f4c4a806737ac 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -32,7 +32,6 @@ ############################################################################ # Modules that should always be present (POSIX and Windows): @MODULE_ARRAY_TRUE@array arraymodule.c -@MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CSV_TRUE@_csv _csv.c @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @@ -56,9 +55,6 @@ @MODULE_CMATH_TRUE@cmath cmathmodule.c @MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c -# needs libm and on some platforms librt -@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c - # _decimal uses libmpdec # either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so # with ./configure --with-system-libmpdec @@ -193,3 +189,9 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c + + +# for performance +*static* + +@MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5f9181395c4828..99408e60721c60 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -821,7 +821,7 @@ future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, Invariants: * callbacks != NULL: - There are some callbacks in in the list. Just + There are some callbacks in the list. Just add the new callback to it. * callbacks == NULL and callback0 == NULL: diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ad670293ec5b6a..3ba48d5d9d3c64 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> @@ -1532,9 +1533,7 @@ deque_dealloc(PyObject *self) Py_ssize_t i; PyObject_GC_UnTrack(deque); - if (deque->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, deque->weakreflist); if (deque->leftblock != NULL) { (void)deque_clear(self); assert(deque->leftblock != NULL); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 7e8a133caa72ac..4bd3e380b3bc4b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3937,7 +3937,9 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) PyObject *name = Py_None; PyObject *defval; PyObject *typ; - if (!PyArg_ParseTuple(item, "i|U?O", &flag, &name, &defval)) { + if (!PyArg_ParseTuple(item, "i|OO", &flag, &name, &defval) || + !(name == Py_None || PyUnicode_Check(name))) + { PyErr_SetString(PyExc_TypeError, "paramflags must be a sequence of (int [,string [,value]]) tuples"); return 0; @@ -4002,8 +4004,10 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) void *handle; PyObject *paramflags = NULL; - if (!PyArg_ParseTuple(args, "O|O?", &ftuple, ¶mflags)) + if (!PyArg_ParseTuple(args, "O|O", &ftuple, ¶mflags)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ftuple = PySequence_Tuple(ftuple); if (!ftuple) @@ -4135,8 +4139,10 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) GUID *iid = NULL; Py_ssize_t iid_len = 0; - if (!PyArg_ParseTuple(args, "is|O?z#", &index, &name, ¶mflags, &iid, &iid_len)) + if (!PyArg_ParseTuple(args, "is|Oz#", &index, &name, ¶mflags, &iid, &iid_len)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5b4f97d43b8721..9f995bf24080a5 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -431,7 +431,7 @@ typedef struct { visible to other threads before the `dict_final` bit is set. */ -#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) +#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUTEX(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() static inline uint8_t diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f208d2956e429e..ab955a0b824a2f 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,7 +484,7 @@ error:; /* Replace array elements at stginfo->ffi_type_pointer.elements. - Return -1 if error occured. + Return -1 if error occurred. */ int _replace_array_elements(ctypes_state *st, PyObject *layout_fields, diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index eb90be81c8d34c..01039dfeec0719 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,7 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() +#include "pycore_initconfig.h" // _PyStatus_OK() #include "datetime.h" @@ -124,10 +125,9 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) static PyObject * -get_current_module(PyInterpreterState *interp, int *p_reloading) +get_current_module(PyInterpreterState *interp) { PyObject *mod = NULL; - int reloading = 0; PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { @@ -138,7 +138,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) goto error; } if (ref != NULL) { - reloading = 1; if (ref != Py_None) { (void)PyWeakref_GetRef(ref, &mod); if (mod == Py_None) { @@ -147,9 +146,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) Py_DECREF(ref); } } - if (p_reloading != NULL) { - *p_reloading = reloading; - } return mod; error: @@ -163,7 +159,7 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); + PyObject *mod = get_current_module(interp); if (mod == NULL) { assert(!PyErr_Occurred()); if (PyErr_Occurred()) { @@ -1934,7 +1930,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } replacement = freplacement; } -#ifdef Py_NORMALIZE_CENTURY +#ifdef _Py_NORMALIZE_CENTURY else if (ch == 'Y' || ch == 'G' || ch == 'F' || ch == 'C' ) { @@ -4482,7 +4478,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base; filled in PyInit__datetime */ + &PyDateTime_TZInfoType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7147,8 +7143,7 @@ static PyTypeObject PyDateTime_DateTimeType = { datetime_methods, /* tp_methods */ 0, /* tp_members */ datetime_getset, /* tp_getset */ - 0, /* tp_base; filled in - PyInit__datetime */ + &PyDateTime_DateType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7329,29 +7324,82 @@ clear_state(datetime_state *st) } -static int -init_static_types(PyInterpreterState *interp, int reloading) +PyStatus +_PyDateTime_InitTypes(PyInterpreterState *interp) { - if (reloading) { - return 0; - } - - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - /* Bases classes must be initialized before subclasses, * so capi_types must have the types in the appropriate order. */ for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; if (_PyStaticType_InitForExtension(interp, type) < 0) { - return -1; + return _PyStatus_ERR("could not initialize static types"); } } - return 0; +#define DATETIME_ADD_MACRO(dict, c, value_expr) \ + do { \ + assert(!PyErr_Occurred()); \ + PyObject *value = (value_expr); \ + if (value == NULL) { \ + goto error; \ + } \ + if (PyDict_SetItemString(dict, c, value) < 0) { \ + Py_DECREF(value); \ + goto error; \ + } \ + Py_DECREF(value); \ + } while(0) + + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } + + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ + + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); + + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO + + return _PyStatus_OK(); + +error: + return _PyStatus_NO_MEMORY(); } @@ -7369,20 +7417,15 @@ _datetime_exec(PyObject *module) { int rc = -1; datetime_state *st = get_module_state(module); - int reloading = 0; PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp, &reloading); + PyObject *old_module = get_current_module(interp); if (PyErr_Occurred()) { assert(old_module == NULL); goto error; } /* We actually set the "current" module right before a successful return. */ - if (init_static_types(interp, reloading) < 0) { - goto error; - } - for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; const char *name = _PyType_Name(type); @@ -7396,68 +7439,6 @@ _datetime_exec(PyObject *module) goto error; } -#define DATETIME_ADD_MACRO(dict, c, value_expr) \ - do { \ - assert(!PyErr_Occurred()); \ - PyObject *value = (value_expr); \ - if (value == NULL) { \ - goto error; \ - } \ - if (PyDict_SetItemString(dict, c, value) < 0) { \ - Py_DECREF(value); \ - goto error; \ - } \ - Py_DECREF(value); \ - } while(0) - - if (!reloading) { - /* timedelta values */ - PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = _PyType_GetDict(&PyDateTime_DateType); - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = _PyType_GetDict(&PyDateTime_TimeType); - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = _PyType_GetDict(&PyDateTime_DateTimeType); - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* timezone values */ - d = _PyType_GetDict(&PyDateTime_TimeZoneType); - if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { - goto error; - } - - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ - - /* -23:59 */ - DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - - /* +23:59 */ - DATETIME_ADD_MACRO( - d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); - } - -#undef DATETIME_ADD_MACRO - /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { goto error; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index fe24629f9f6f70..b9e12ab2026f65 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -17,6 +17,7 @@ #include "Python.h" #include "pycore_pyhash.h" // _Py_HashSecret +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "expat.h" @@ -690,8 +691,7 @@ element_dealloc(PyObject *op) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); /* element_gc_clear clears all references and deallocates extra */ diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 354dbad84b5099..1c888295cb07f1 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -7,6 +7,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/_functoolsmodule.c.h" @@ -351,9 +352,7 @@ partial_dealloc(PyObject *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (partialobject_CAST(self)->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, partialobject_CAST(self)->weakreflist); (void)partial_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -368,19 +367,6 @@ partial_descr_get(PyObject *self, PyObject *obj, PyObject *type) return PyMethod_New(self, obj); } -/* Merging keyword arguments using the vectorcall convention is messy, so - * if we would need to do that, we stop using vectorcall and fall back - * to using partial_call() instead. */ -Py_NO_INLINE static PyObject * -partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, - PyObject *const *args, size_t nargsf, - PyObject *kwnames) -{ - pto->vectorcall = NULL; - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); -} - static PyObject * partial_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -389,10 +375,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - /* pto->kw is mutable, so need to check every time */ - if (PyDict_GET_SIZE(pto->kw)) { - return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); - } + /* Placeholder check */ Py_ssize_t pto_phcount = pto->phcount; if (nargs < pto_phcount) { PyErr_Format(PyExc_TypeError, @@ -401,50 +384,143 @@ partial_vectorcall(PyObject *self, PyObject *const *args, return NULL; } - Py_ssize_t nargskw = nargs; - if (kwnames != NULL) { - nargskw += PyTuple_GET_SIZE(kwnames); - } - PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t pto_nkwds = PyDict_GET_SIZE(pto->kw); + Py_ssize_t nkwds = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t nargskw = nargs + nkwds; + + /* Special cases */ + if (!pto_nkwds) { + /* Fast path if we're called without arguments */ + if (nargskw == 0) { + return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, + pto_nargs, NULL); + } - /* Fast path if we're called without arguments */ - if (nargskw == 0) { - return _PyObject_VectorcallTstate(tstate, pto->fn, - pto_args, pto_nargs, NULL); + /* Use PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single + * positional argument. */ + if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { + PyObject **newargs = (PyObject **)args - 1; + PyObject *tmp = newargs[0]; + newargs[0] = pto_args[0]; + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, newargs, + nargs + 1, kwnames); + newargs[0] = tmp; + return ret; + } } - /* Fast path using PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single - * positional argument */ - if (pto_nargs == 1 && (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET)) { - PyObject **newargs = (PyObject **)args - 1; - PyObject *tmp = newargs[0]; - newargs[0] = pto_args[0]; - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, - newargs, nargs + 1, kwnames); - newargs[0] = tmp; - return ret; - } + /* Total sizes */ + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + Py_ssize_t tot_nkwds = pto_nkwds + nkwds; + Py_ssize_t tot_nargskw = tot_nargs + tot_nkwds; - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject **stack; + PyObject *pto_kw_merged = NULL; // pto_kw with duplicates merged (if any) + PyObject *tot_kwnames; - Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; - if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + /* Allocate Stack + * Note, _PY_FASTCALL_SMALL_STACK is optimal for positional only + * This case might have keyword arguments + * furthermore, it might use extra stack space for temporary key storage + * thus, double small_stack size is used, which is 10 * 8 = 80 bytes */ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK * 2]; + PyObject **tmp_stack, **stack; + Py_ssize_t init_stack_size = tot_nargskw; + if (pto_nkwds) { + // If pto_nkwds, allocate additional space for temporary new keys + init_stack_size += nkwds; + } + if (init_stack_size <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); + stack = PyMem_Malloc(init_stack_size * sizeof(PyObject *)); if (stack == NULL) { - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } } - Py_ssize_t tot_nargs; + /* Copy keywords to stack */ + if (!pto_nkwds) { + tot_kwnames = kwnames; + if (nkwds) { + /* if !pto_nkwds & nkwds, then simply append kw */ + memcpy(stack + tot_nargs, args + nargs, nkwds * sizeof(PyObject*)); + } + } + else { + /* stack is now [<positionals>, <pto_kwds>, <kwds>, <kwds_keys>] + * Will resize later to [<positionals>, <merged_kwds>] */ + PyObject *key, *val; + + /* Merge kw to pto_kw or add to tail (if not duplicate) */ + Py_ssize_t n_tail = 0; + for (Py_ssize_t i = 0; i < nkwds; ++i) { + key = PyTuple_GET_ITEM(kwnames, i); + val = args[nargs + i]; + if (PyDict_Contains(pto->kw, key)) { + if (pto_kw_merged == NULL) { + pto_kw_merged = PyDict_Copy(pto->kw); + if (pto_kw_merged == NULL) { + goto error; + } + } + if (PyDict_SetItem(pto_kw_merged, key, val) < 0) { + Py_DECREF(pto_kw_merged); + goto error; + } + } + else { + /* Copy keyword tail to stack */ + stack[tot_nargs + pto_nkwds + n_tail] = val; + stack[tot_nargskw + n_tail] = key; + n_tail++; + } + } + Py_ssize_t n_merges = nkwds - n_tail; + + /* Create total kwnames */ + tot_kwnames = PyTuple_New(tot_nkwds - n_merges); + if (tot_kwnames == NULL) { + Py_XDECREF(pto_kw_merged); + goto error; + } + for (Py_ssize_t i = 0; i < n_tail; ++i) { + key = Py_NewRef(stack[tot_nargskw + i]); + PyTuple_SET_ITEM(tot_kwnames, pto_nkwds + i, key); + } + + /* Copy pto_keywords with overlapping call keywords merged + * Note, tail is already coppied. */ + Py_ssize_t pos = 0, i = 0; + while (PyDict_Next(n_merges ? pto_kw_merged : pto->kw, &pos, &key, &val)) { + assert(i < pto_nkwds); + PyTuple_SET_ITEM(tot_kwnames, i, Py_NewRef(key)); + stack[tot_nargs + i] = val; + i++; + } + assert(i == pto_nkwds); + Py_XDECREF(pto_kw_merged); + + /* Resize Stack if the removing overallocation saves some noticable memory + * NOTE: This whole block can be removed without breaking anything */ + Py_ssize_t noveralloc = n_merges + nkwds; + if (stack != small_stack && noveralloc > 6 && noveralloc > init_stack_size / 10) { + tmp_stack = PyMem_Realloc(stack, (tot_nargskw - n_merges) * sizeof(PyObject *)); + if (tmp_stack == NULL) { + Py_DECREF(tot_kwnames); + if (stack != small_stack) { + PyMem_Free(stack); + } + return PyErr_NoMemory(); + } + stack = tmp_stack; + } + } + + /* Copy Positionals to stack */ if (pto_phcount) { - tot_nargs = pto_nargs + nargs - pto_phcount; Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { if (pto_args[i] == pto->placeholder) { @@ -456,22 +532,31 @@ partial_vectorcall(PyObject *self, PyObject *const *args, } } assert(j == pto_phcount); - if (nargskw > pto_phcount) { - memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); + /* Add remaining args from new_args */ + if (nargs > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargs - j) * sizeof(PyObject*)); } } else { - tot_nargs = pto_nargs + nargs; - /* Copy to new stack, using borrowed references */ memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargs * sizeof(PyObject*)); } - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, tot_nargs, kwnames); + + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, + tot_nargs, tot_kwnames); if (stack != small_stack) { PyMem_Free(stack); } + if (pto_nkwds) { + Py_DECREF(tot_kwnames); + } return ret; + + error: + if (stack != small_stack) { + PyMem_Free(stack); + } + return NULL; } /* Set pto->vectorcall depending on the parameters of the partial object */ @@ -1621,9 +1706,7 @@ lru_cache_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(obj); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(obj); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, obj->weakreflist); (void)lru_cache_tp_clear(op); tp->tp_free(obj); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 90a7391ebb09af..d79e4b360e95c5 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -48,7 +48,6 @@ #define MUNCH_SIZE INT_MAX -#define PY_OPENSSL_HAS_SCRYPT 1 #if defined(NID_sha3_224) && defined(NID_sha3_256) && defined(NID_sha3_384) && defined(NID_sha3_512) #define PY_OPENSSL_HAS_SHA3 1 #endif @@ -64,13 +63,29 @@ #define PY_EVP_MD_fetch(algorithm, properties) EVP_MD_fetch(NULL, algorithm, properties) #define PY_EVP_MD_up_ref(md) EVP_MD_up_ref(md) #define PY_EVP_MD_free(md) EVP_MD_free(md) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_get0_md(CTX) #else #define PY_EVP_MD const EVP_MD #define PY_EVP_MD_fetch(algorithm, properties) EVP_get_digestbyname(algorithm) #define PY_EVP_MD_up_ref(md) do {} while(0) #define PY_EVP_MD_free(md) do {} while(0) + +#define PY_EVP_MD_CTX_md(CTX) EVP_MD_CTX_md(CTX) #endif +/* + * Return 1 if *md* is an extendable-output Function (XOF) and 0 otherwise. + * SHAKE128 and SHAKE256 are XOF functions but not BLAKE2B algorithms. + * + * This is a backport of the EVP_MD_xof() helper added in OpenSSL 3.4. + */ +static inline int +PY_EVP_MD_xof(PY_EVP_MD *md) +{ + return md != NULL && ((EVP_MD_flags(md) & EVP_MD_FLAG_XOF) != 0); +} + /* hash alias map and fast lookup * * Map between Python's preferred names and OpenSSL internal names. Maintain @@ -308,6 +323,43 @@ class _hashlib.HMAC "HMACobject *" "&PyType_Type" /* LCOV_EXCL_START */ +/* Thin wrapper around ERR_reason_error_string() returning non-NULL text. */ +static const char * +py_wrapper_ERR_reason_error_string(unsigned long errcode) +{ + const char *reason = ERR_reason_error_string(errcode); + return reason ? reason : "no reason"; +} + +#ifdef Py_HAS_OPENSSL3_SUPPORT +/* + * Set an exception with additional information. + * + * This is only useful in OpenSSL 3.0 and later as the default reason + * usually lacks information and function locations are no longer encoded + * in the error code. + */ +static void +set_exception_with_ssl_errinfo(PyObject *exc_type, PyObject *exc_text, + const char *lib, const char *reason) +{ + assert(exc_type != NULL); + assert(exc_text != NULL); + if (lib && reason) { + PyErr_Format(exc_type, "[%s] %U (reason: %s)", lib, exc_text, reason); + } + else if (lib) { + PyErr_Format(exc_type, "[%s] %U", lib, exc_text); + } + else if (reason) { + PyErr_Format(exc_type, "%U (reason: %s)", exc_text, reason); + } + else { + PyErr_SetObject(exc_type, exc_text); + } +} +#endif + /* Set an exception of given type using the given OpenSSL error code. */ static void set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) @@ -317,8 +369,13 @@ set_ssl_exception_from_errcode(PyObject *exc_type, unsigned long errcode) /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ const char *lib = ERR_lib_error_string(errcode); +#ifdef Py_HAS_OPENSSL3_SUPPORT + // Since OpenSSL 3.0, ERR_func_error_string() always returns NULL. + const char *func = NULL; +#else const char *func = ERR_func_error_string(errcode); - const char *reason = ERR_reason_error_string(errcode); +#endif + const char *reason = py_wrapper_ERR_reason_error_string(errcode); if (lib && func) { PyErr_Format(exc_type, "[%s: %s] %s", lib, func, reason); @@ -429,6 +486,68 @@ notify_smart_ssl_error_occurred_in(const char *funcname) raise_smart_ssl_error_f(PyExc_ValueError, "error in OpenSSL function %s()", funcname); } + +#ifdef Py_HAS_OPENSSL3_SUPPORT +static void +raise_unsupported_algorithm_impl(PyObject *exc_type, + const char *fallback_format, + const void *format_arg) +{ + // Since OpenSSL 3.0, if the algorithm is not supported or fetching fails, + // the reason lacks the algorithm name. + int errcode = ERR_peek_last_error(); + switch (ERR_GET_REASON(errcode)) { + case ERR_R_UNSUPPORTED: { + PyObject *text = PyUnicode_FromFormat(fallback_format, format_arg); + if (text != NULL) { + const char *lib = ERR_lib_error_string(errcode); + set_exception_with_ssl_errinfo(exc_type, text, lib, NULL); + Py_DECREF(text); + } + break; + } + case ERR_R_FETCH_FAILED: { + PyObject *text = PyUnicode_FromFormat(fallback_format, format_arg); + if (text != NULL) { + const char *lib = ERR_lib_error_string(errcode); + const char *reason = ERR_reason_error_string(errcode); + set_exception_with_ssl_errinfo(exc_type, text, lib, reason); + Py_DECREF(text); + } + break; + } + default: + raise_ssl_error_f(exc_type, fallback_format, format_arg); + break; + } + assert(PyErr_Occurred()); +} +#else +/* Before OpenSSL 3.0, error messages included enough information. */ +#define raise_unsupported_algorithm_impl raise_ssl_error_f +#endif + +static inline void +raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) +{ + raise_unsupported_algorithm_impl( + state->unsupported_digestmod_error, + HASHLIB_UNSUPPORTED_ALGORITHM, + digestmod + ); +} + +static inline void +raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) +{ + raise_unsupported_algorithm_impl( + state->unsupported_digestmod_error, + HASHLIB_UNSUPPORTED_STR_ALGORITHM, + name + ); +} + +#undef raise_unsupported_algorithm_impl /* LCOV_EXCL_STOP */ /* @@ -506,26 +625,40 @@ get_hashlib_utf8name_by_evp_md(const EVP_MD *md) return get_hashlib_utf8name_by_nid(EVP_MD_nid(md)); } +/* + * Return 1 if the property query clause [1] must be "-fips" and 0 otherwise. + * + * [1] https://docs.openssl.org/master/man7/property + */ +static inline int +disable_fips_property(Py_hash_type py_ht) +{ + switch (py_ht) { + case Py_ht_evp: + case Py_ht_mac: + case Py_ht_pbkdf2: + return 0; + case Py_ht_evp_nosecurity: + return 1; + default: + Py_FatalError("unsupported hash type"); + } +} + /* * Get a new reference to an EVP_MD object described by name and purpose. * * If 'name' is an OpenSSL indexed name, the return value is cached. */ static PY_EVP_MD * -get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, +get_openssl_evp_md_by_utf8name(_hashlibstate *state, const char *name, Py_hash_type py_ht) { PY_EVP_MD *digest = NULL, *other_digest = NULL; - _hashlibstate *state = get_hashlib_state(module); - py_hashentry_t *entry = (py_hashentry_t *)_Py_hashtable_get( - state->hashtable, (const void*)name - ); + py_hashentry_t *entry = _Py_hashtable_get(state->hashtable, name); if (entry != NULL) { - switch (py_ht) { - case Py_ht_evp: - case Py_ht_mac: - case Py_ht_pbkdf2: + if (!disable_fips_property(py_ht)) { digest = FT_ATOMIC_LOAD_PTR_RELAXED(entry->evp); if (digest == NULL) { digest = PY_EVP_MD_fetch(entry->ossl_name, NULL); @@ -536,8 +669,8 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, entry->evp = digest; #endif } - break; - case Py_ht_evp_nosecurity: + } + else { digest = FT_ATOMIC_LOAD_PTR_RELAXED(entry->evp_nosecurity); if (digest == NULL) { digest = PY_EVP_MD_fetch(entry->ossl_name, "-fips"); @@ -548,9 +681,6 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, entry->evp_nosecurity = digest; #endif } - break; - default: - goto invalid_hash_type; } // if another thread same thing at same time make sure we got same ptr assert(other_digest == NULL || other_digest == digest); @@ -560,41 +690,15 @@ get_openssl_evp_md_by_utf8name(PyObject *module, const char *name, } else { // Fall back for looking up an unindexed OpenSSL specific name. - switch (py_ht) { - case Py_ht_evp: - case Py_ht_mac: - case Py_ht_pbkdf2: - digest = PY_EVP_MD_fetch(name, NULL); - break; - case Py_ht_evp_nosecurity: - digest = PY_EVP_MD_fetch(name, "-fips"); - break; - default: - goto invalid_hash_type; - } + const char *props = disable_fips_property(py_ht) ? "-fips" : NULL; + (void)props; // will only be used in OpenSSL 3.0 and later + digest = PY_EVP_MD_fetch(name, props); } if (digest == NULL) { - raise_ssl_error_f(state->unsupported_digestmod_error, - "unsupported digest name: %s", name); + raise_unsupported_str_algorithm_error(state, name); return NULL; } return digest; - -invalid_hash_type: - assert(digest == NULL); - PyErr_Format(PyExc_SystemError, "unsupported hash type %d", py_ht); - return NULL; -} - -/* - * Raise an exception indicating that 'digestmod' is not supported. - */ -static void -raise_unsupported_digestmod_error(PyObject *module, PyObject *digestmod) -{ - _hashlibstate *state = get_hashlib_state(module); - PyErr_Format(state->unsupported_digestmod_error, - "Unsupported digestmod %R", digestmod); } /* @@ -608,25 +712,25 @@ raise_unsupported_digestmod_error(PyObject *module, PyObject *digestmod) * py_ht The message digest purpose. */ static PY_EVP_MD * -get_openssl_evp_md(PyObject *module, PyObject *digestmod, Py_hash_type py_ht) +get_openssl_evp_md(_hashlibstate *state, PyObject *digestmod, Py_hash_type py_ht) { const char *name; if (PyUnicode_Check(digestmod)) { name = PyUnicode_AsUTF8(digestmod); } else { - PyObject *dict = get_hashlib_state(module)->constructs; + PyObject *dict = state->constructs; assert(dict != NULL); PyObject *borrowed_ref = PyDict_GetItemWithError(dict, digestmod); name = borrowed_ref == NULL ? NULL : PyUnicode_AsUTF8(borrowed_ref); } if (name == NULL) { if (!PyErr_Occurred()) { - raise_unsupported_digestmod_error(module, digestmod); + raise_unsupported_algorithm_error(state, digestmod); } return NULL; } - return get_openssl_evp_md_by_utf8name(module, name, py_ht); + return get_openssl_evp_md_by_utf8name(state, name, py_ht); } // --- OpenSSL HASH wrappers -------------------------------------------------- @@ -838,7 +942,7 @@ static PyObject * _hashlib_HASH_get_name(PyObject *op, void *Py_UNUSED(closure)) { HASHobject *self = HASHobject_CAST(op); - const EVP_MD *md = EVP_MD_CTX_md(self->ctx); + const EVP_MD *md = PY_EVP_MD_CTX_md(self->ctx); if (md == NULL) { notify_ssl_error_occurred("missing EVP_MD for HASH context"); return NULL; @@ -1083,7 +1187,7 @@ static PyType_Spec HASHXOFobject_type_spec = { #endif static PyObject * -_hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, +_hashlib_HASH(_hashlibstate *state, const char *digestname, PyObject *data_obj, int usedforsecurity) { Py_buffer view = { 0 }; @@ -1095,19 +1199,13 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); } - digest = get_openssl_evp_md_by_utf8name( - module, digestname, usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity - ); + Py_hash_type purpose = usedforsecurity ? Py_ht_evp : Py_ht_evp_nosecurity; + digest = get_openssl_evp_md_by_utf8name(state, digestname, purpose); if (digest == NULL) { goto exit; } - if ((EVP_MD_flags(digest) & EVP_MD_FLAG_XOF) == EVP_MD_FLAG_XOF) { - type = get_hashlib_state(module)->HASHXOF_type; - } else { - type = get_hashlib_state(module)->HASH_type; - } - + type = PY_EVP_MD_xof(digest) ? state->HASHXOF_type : state->HASH_type; self = new_hash_object(type); if (self == NULL) { goto exit; @@ -1159,7 +1257,8 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, if (_Py_hashlib_data_argument(&data_obj, DATA, STRING) < 0) { \ return NULL; \ } \ - return _hashlib_HASH(MODULE, NAME, data_obj, USEDFORSECURITY); \ + _hashlibstate *state = get_hashlib_state(MODULE); \ + return _hashlib_HASH(state, NAME, data_obj, USEDFORSECURITY); \ } while (0) /* The module-level function: new() */ @@ -1461,12 +1560,14 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, PyObject *dklen_obj) /*[clinic end generated code: output=144b76005416599b input=ed3ab0d2d28b5d5c]*/ { + _hashlibstate *state = get_hashlib_state(module); PyObject *key_obj = NULL; char *key; long dklen; int retval; - PY_EVP_MD *digest = get_openssl_evp_md_by_utf8name(module, hash_name, Py_ht_pbkdf2); + PY_EVP_MD *digest = get_openssl_evp_md_by_utf8name(state, hash_name, + Py_ht_pbkdf2); if (digest == NULL) { goto end; } @@ -1540,7 +1641,25 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, return key_obj; } -#ifdef PY_OPENSSL_HAS_SCRYPT +// --- PBKDF: scrypt (RFC 7914) ----------------------------------------------- + +/* + * By default, OpenSSL 1.1.0 restricts 'maxmem' in EVP_PBE_scrypt() + * to 32 MiB (1024 * 1024 * 32) but only if 'maxmem = 0' and allows + * for an arbitrary large limit fitting on an uint64_t otherwise. + * + * For legacy reasons, we limited 'maxmem' to be at most INTMAX, + * but if users need a more relaxed value, we will revisit this + * limit in the future. + */ +#define HASHLIB_SCRYPT_MAX_MAXMEM INT_MAX + +/* + * Limit 'dklen' to INT_MAX even if it can be at most (32 * UINT32_MAX). + * + * See https://datatracker.ietf.org/doc/html/rfc7914.html for details. + */ +#define HASHLIB_SCRYPT_MAX_DKLEN INT_MAX /*[clinic input] _hashlib.scrypt @@ -1554,7 +1673,6 @@ _hashlib.scrypt maxmem: long = 0 dklen: long = 64 - scrypt password-based key derivation function. [clinic start generated code]*/ @@ -1562,77 +1680,73 @@ static PyObject * _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, unsigned long n, unsigned long r, unsigned long p, long maxmem, long dklen) -/*[clinic end generated code: output=d424bc3e8c6b9654 input=0c9a84230238fd79]*/ +/*[clinic end generated code: output=d424bc3e8c6b9654 input=bdeac9628d07f7a1]*/ { - PyObject *key_obj = NULL; - char *key; + PyObject *key = NULL; int retval; if (password->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "password is too long."); + PyErr_SetString(PyExc_OverflowError, "password is too long"); return NULL; } if (salt->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "salt is too long."); + PyErr_SetString(PyExc_OverflowError, "salt is too long"); return NULL; } if (n < 2 || n & (n - 1)) { - PyErr_SetString(PyExc_ValueError, - "n must be a power of 2."); + PyErr_SetString(PyExc_ValueError, "n must be a power of 2"); return NULL; } - if (maxmem < 0 || maxmem > INT_MAX) { - /* OpenSSL 1.1.0 restricts maxmem to 32 MiB. It may change in the - future. The maxmem constant is private to OpenSSL. */ + if (maxmem < 0 || maxmem > HASHLIB_SCRYPT_MAX_MAXMEM) { PyErr_Format(PyExc_ValueError, - "maxmem must be positive and smaller than %d", - INT_MAX); + "maxmem must be positive and at most %d", + HASHLIB_SCRYPT_MAX_MAXMEM); return NULL; } - if (dklen < 1 || dklen > INT_MAX) { + if (dklen < 1 || dklen > HASHLIB_SCRYPT_MAX_DKLEN) { PyErr_Format(PyExc_ValueError, - "dklen must be greater than 0 and smaller than %d", - INT_MAX); + "dklen must be at least 1 and at most %d", + HASHLIB_SCRYPT_MAX_DKLEN); return NULL; } /* let OpenSSL validate the rest */ - retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0); + retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, + (uint64_t)maxmem, NULL, 0); if (!retval) { - notify_ssl_error_occurred( - "Invalid parameter combination for n, r, p, maxmem."); + notify_ssl_error_occurred("invalid parameter combination for " + "n, r, p, and maxmem"); return NULL; } - key_obj = PyBytes_FromStringAndSize(NULL, dklen); - if (key_obj == NULL) { + key = PyBytes_FromStringAndSize(NULL, dklen); + if (key == NULL) { return NULL; } - key = PyBytes_AS_STRING(key_obj); Py_BEGIN_ALLOW_THREADS retval = EVP_PBE_scrypt( - (const char*)password->buf, (size_t)password->len, + (const char *)password->buf, (size_t)password->len, (const unsigned char *)salt->buf, (size_t)salt->len, - n, r, p, maxmem, - (unsigned char *)key, (size_t)dklen + (uint64_t)n, (uint64_t)r, (uint64_t)p, (uint64_t)maxmem, + (unsigned char *)PyBytes_AS_STRING(key), (size_t)dklen ); Py_END_ALLOW_THREADS if (!retval) { - Py_CLEAR(key_obj); + Py_DECREF(key); notify_ssl_error_occurred_in(Py_STRINGIFY(EVP_PBE_scrypt)); return NULL; } - return key_obj; + return key; } -#endif /* PY_OPENSSL_HAS_SCRYPT */ + +#undef HASHLIB_SCRYPT_MAX_DKLEN +#undef HASHLIB_SCRYPT_MAX_MAXMEM /* Fast HMAC for hmac.digest() */ @@ -1652,10 +1766,12 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, Py_buffer *msg, PyObject *digest) /*[clinic end generated code: output=82f19965d12706ac input=0a0790cc3db45c2e]*/ { + _hashlibstate *state = get_hashlib_state(module); unsigned char md[EVP_MAX_MD_SIZE] = {0}; unsigned int md_len = 0; unsigned char *result; PY_EVP_MD *evp; + int is_xof; if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, @@ -1668,12 +1784,13 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, return NULL; } - evp = get_openssl_evp_md(module, digest, Py_ht_mac); + evp = get_openssl_evp_md(state, digest, Py_ht_mac); if (evp == NULL) { return NULL; } Py_BEGIN_ALLOW_THREADS + is_xof = PY_EVP_MD_xof(evp); result = HMAC( evp, (const void *)key->buf, (int)key->len, @@ -1684,7 +1801,13 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, PY_EVP_MD_free(evp); if (result == NULL) { - notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC)); + if (is_xof) { + /* use a better default error message if an XOF is used */ + raise_unsupported_algorithm_error(state, digest); + } + else { + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC)); + } return NULL; } return PyBytes_FromStringAndSize((const char*)md, md_len); @@ -1732,10 +1855,11 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, PyObject *digestmod) /*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/ { + _hashlibstate *state = get_hashlib_state(module); PY_EVP_MD *digest; HMAC_CTX *ctx = NULL; HMACobject *self = NULL; - int r; + int is_xof, r; if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, @@ -1749,7 +1873,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - digest = get_openssl_evp_md(module, digestmod, Py_ht_mac); + digest = get_openssl_evp_md(state, digestmod, Py_ht_mac); if (digest == NULL) { return NULL; } @@ -1760,14 +1884,20 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, goto error; } + is_xof = PY_EVP_MD_xof(digest); r = HMAC_Init_ex(ctx, key->buf, (int)key->len, digest, NULL /* impl */); PY_EVP_MD_free(digest); if (r == 0) { - notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Init_ex)); + if (is_xof) { + /* use a better default error message if an XOF is used */ + raise_unsupported_algorithm_error(state, digestmod); + } + else { + notify_ssl_error_occurred_in(Py_STRINGIFY(HMAC_Init_ex)); + } goto error; } - _hashlibstate *state = get_hashlib_state(module); self = PyObject_New(HMACobject, state->HMAC_type); if (self == NULL) { goto error; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 560fe431fcac99..05d01acd77109b 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -463,11 +463,11 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } - /* Bubble up the smaller child until hitting a leaf. */ + /* Bubble up the larger child until hitting a leaf. */ arr = _PyList_ITEMS(heap); limit = endpos >> 1; /* smallest pos that has no child */ while (pos < limit) { - /* Set childpos to index of smaller child. */ + /* Set childpos to index of larger child. */ childpos = 2*pos + 1; /* leftmost child position */ if (childpos + 1 < endpos) { PyObject* a = arr[childpos + 1]; @@ -487,7 +487,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } } - /* Move the smaller child up. */ + /* Move the larger child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index e7feaa7f186aee..faf3b25b68c4eb 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -666,7 +666,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables, &result) < 0) { - // If an error occured at this step, it means that interp + // If an error occurred at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); _PyXI_FreeFailure(failure); @@ -1415,11 +1415,14 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) PyObject *idobj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O?|$p:get_config", kwlist, + "O|$p:get_config", kwlist, &idobj, &restricted)) { return NULL; } + if (idobj == Py_None) { + idobj = NULL; + } int reqready = 0; PyInterpreterState *interp = \ @@ -1536,14 +1539,14 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"exc", NULL}; PyObject *exc_arg = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O?:capture_exception", kwlist, + "|O:capture_exception", kwlist, &exc_arg)) { return NULL; } PyObject *exc = exc_arg; - if (exc == NULL) { + if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); if (exc == NULL) { Py_RETURN_NONE; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 4724e97982f349..25c8bf8b3d508b 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -13,6 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -421,8 +422,7 @@ buffered_dealloc(PyObject *op) return; _PyObject_GC_UNTRACK(self); self->ok = 0; - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -2312,8 +2312,7 @@ bufferedrwpair_dealloc(PyObject *op) rwpair *self = rwpair_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)bufferedrwpair_clear(op); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 61cfec435fe60f..30d61f9d68e610 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -3,6 +3,7 @@ #include "pycore_object.h" #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "_iomodule.h" @@ -269,6 +270,7 @@ bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) } /*[clinic input] +@critical_section _io.BytesIO.readable Returns True if the IO object can be read. @@ -276,13 +278,14 @@ Returns True if the IO object can be read. static PyObject * _io_BytesIO_readable_impl(bytesio *self) -/*[clinic end generated code: output=4e93822ad5b62263 input=96c5d0cccfb29f5c]*/ +/*[clinic end generated code: output=4e93822ad5b62263 input=ab7816facef48bfd]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.writable Returns True if the IO object can be written. @@ -290,13 +293,14 @@ Returns True if the IO object can be written. static PyObject * _io_BytesIO_writable_impl(bytesio *self) -/*[clinic end generated code: output=64ff6a254b1150b8 input=700eed808277560a]*/ +/*[clinic end generated code: output=64ff6a254b1150b8 input=4f35d49d26dab024]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.seekable Returns True if the IO object can be seeked. @@ -304,13 +308,14 @@ Returns True if the IO object can be seeked. static PyObject * _io_BytesIO_seekable_impl(bytesio *self) -/*[clinic end generated code: output=6b417f46dcc09b56 input=9421f65627a344dd]*/ +/*[clinic end generated code: output=6b417f46dcc09b56 input=9cc78d15aa1deaa3]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.flush Does nothing. @@ -318,7 +323,7 @@ Does nothing. static PyObject * _io_BytesIO_flush_impl(bytesio *self) -/*[clinic end generated code: output=187e3d781ca134a0 input=561ea490be4581a7]*/ +/*[clinic end generated code: output=187e3d781ca134a0 input=c60842743910b381]*/ { CHECK_CLOSED(self); Py_RETURN_NONE; @@ -384,6 +389,7 @@ _io_BytesIO_getvalue_impl(bytesio *self) } /*[clinic input] +@critical_section _io.BytesIO.isatty Always returns False. @@ -393,7 +399,7 @@ BytesIO objects are not connected to a TTY-like device. static PyObject * _io_BytesIO_isatty_impl(bytesio *self) -/*[clinic end generated code: output=df67712e669f6c8f input=6f97f0985d13f827]*/ +/*[clinic end generated code: output=df67712e669f6c8f input=50487b74dc5ae8a9]*/ { CHECK_CLOSED(self); Py_RETURN_FALSE; @@ -979,8 +985,7 @@ bytesio_dealloc(PyObject *op) } Py_CLEAR(self->buf); Py_CLEAR(self->dict); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 8553ed05f70d1b..6595dc937bbcf0 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -25,7 +25,13 @@ _io_BytesIO_readable_impl(bytesio *self); static PyObject * _io_BytesIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_readable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_readable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_writable__doc__, @@ -43,7 +49,13 @@ _io_BytesIO_writable_impl(bytesio *self); static PyObject * _io_BytesIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_writable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_writable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_seekable__doc__, @@ -61,7 +73,13 @@ _io_BytesIO_seekable_impl(bytesio *self); static PyObject * _io_BytesIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_seekable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_seekable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_flush__doc__, @@ -79,7 +97,13 @@ _io_BytesIO_flush_impl(bytesio *self); static PyObject * _io_BytesIO_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_flush_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_flush_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, @@ -152,7 +176,13 @@ _io_BytesIO_isatty_impl(bytesio *self); static PyObject * _io_BytesIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_isatty_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_isatty_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_tell__doc__, @@ -607,4 +637,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=580205daa01def2e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=daa81dfdae5ccc57 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8fcb27049d6c7c..26537fc6395e9f 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -4,6 +4,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> // bool #ifdef HAVE_UNISTD_H @@ -570,9 +571,7 @@ fileio_dealloc(PyObject *op) PyMem_Free(self->stat_atopen); self->stat_atopen = NULL; } - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)fileio_clear(op); PyTypeObject *tp = Py_TYPE(op); diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index cd4c7e7cead277..044f6b7803c571 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -14,6 +14,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "_iomodule.h" @@ -383,8 +384,7 @@ iobase_dealloc(PyObject *op) } PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 56913fafefba8b..20b7cfc00880cd 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,6 +1,7 @@ #include "Python.h" #include <stddef.h> // offsetof() #include "pycore_object.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" /* Implementation note: the buffer is always at least one character longer @@ -638,9 +639,7 @@ stringio_dealloc(PyObject *op) } PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3808ecdceb9b70..5354cf63442599 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -16,6 +16,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_unicodeobject.h" // _PyUnicode_AsASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -1469,8 +1470,7 @@ textiowrapper_dealloc(PyObject *op) return; self->ok = 0; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)textiowrapper_clear(op); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 3e783b9da45652..950b7fe241ccde 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -10,6 +10,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #ifdef HAVE_WINDOWS_CONSOLE_IO @@ -518,8 +519,7 @@ winconsoleio_dealloc(PyObject *op) if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_json.c b/Modules/_json.c index 6b5f6ea42df4d1..7580b589e2d937 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1222,16 +1222,23 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; - PyObject *markers = Py_None, *defaultfn, *encoder, *indent, *key_separator; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!?OOOUUppp:make_encoder", kwlist, - &PyDict_Type, &markers, &defaultfn, &encoder, &indent, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) return NULL; + if (markers != Py_None && !PyDict_Check(markers)) { + PyErr_Format(PyExc_TypeError, + "make_encoder() argument 1 must be dict or None, " + "not %.200s", Py_TYPE(markers)->tp_name); + return NULL; + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index c1f56008b7c49e..41e6d48b1dbd9b 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -692,7 +692,7 @@ _locale_nl_langinfo_impl(PyObject *module, int item) result = result != NULL ? result : ""; char *oldloc = NULL; if (langinfo_constants[i].category != LC_CTYPE - && ( + && *result && ( #ifdef __GLIBC__ // gh-133740: Always change the locale for ALT_DIGITS and ERA # ifdef ALT_DIGITS diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 3ee14b61b821d6..01235c77bd7db8 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" #include "pycore_time.h" // _PyTime_FromSecondsObject() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -221,9 +222,7 @@ simplequeue_dealloc(PyObject *op) PyObject_GC_UnTrack(self); (void)simplequeue_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index ce7189637c2d69..b50e5e403a1a19 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -257,20 +257,25 @@ is_frame_valid( uintptr_t code_object_addr ); -static int -parse_tasks_in_set( +typedef int (*thread_processor_func)( RemoteUnwinderObject *unwinder, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); + +typedef int (*set_entry_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context ); + static int parse_task( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *render_to, - int recurse_task + PyObject *render_to ); static int @@ -285,9 +290,27 @@ static int parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, + uintptr_t* address_of_code_object, uintptr_t* previous_frame ); +static int +parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +); + +static int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr); +static int read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *size); + +static int process_task_and_waiters(RemoteUnwinderObject *unwinder, uintptr_t task_addr, PyObject *result); +static int process_task_awaited_by(RemoteUnwinderObject *unwinder, uintptr_t task_address, set_entry_processor_func processor, void *context); +static int find_running_task_in_thread(RemoteUnwinderObject *unwinder, uintptr_t thread_state_addr, uintptr_t *running_task_addr); +static int get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr); +static int append_awaited_by(RemoteUnwinderObject *unwinder, unsigned long tid, uintptr_t head_addr, PyObject *result); + /* ============================================================================ * UTILITY FUNCTIONS AND HELPERS * ============================================================================ */ @@ -405,32 +428,149 @@ validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets) return 0; } -/* ============================================================================ - * MEMORY READING FUNCTIONS - * ============================================================================ */ +// Generic function to iterate through all threads +static int +iterate_threads( + RemoteUnwinderObject *unwinder, + thread_processor_func processor, + void *context +) { + uintptr_t thread_state_addr; + unsigned long tid = 0; -static inline int -read_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) -{ - int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(void*), ptr_addr); - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read pointer from remote memory"); + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + unwinder->interpreter_addr + unwinder->debug_offsets.interpreter_state.threads_main, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); return -1; } + + while (thread_state_addr != 0) { + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + unwinder->debug_offsets.thread_state.native_thread_id, + sizeof(tid), + &tid)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread ID"); + return -1; + } + + // Call the processor function for this thread + if (processor(unwinder, thread_state_addr, tid, context) < 0) { + return -1; + } + + // Move to next thread + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next thread state"); + return -1; + } + } + return 0; } -static inline int -read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *size) -{ - int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(Py_ssize_t), size); - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read Py_ssize_t from remote memory"); +// Generic function to iterate through set entries +static int +iterate_set_entries( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + set_entry_processor_func processor, + void *context +) { + char set_object[SIZEOF_SET_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, set_addr, + SIZEOF_SET_OBJ, set_object) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); return -1; } + + Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); + Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; + uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len && els < num_els) { + uintptr_t key_addr; + if (read_py_ptr(unwinder, table_ptr, &key_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry ref count"); + return -1; + } + + if (ref_cnt) { + // Process this valid set entry + if (processor(unwinder, key_addr, context) < 0) { + return -1; + } + els++; + } + } + table_ptr += sizeof(void*) * 2; + i++; + } + return 0; } +// Processor function for task waiters +static int +process_waiter_task( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *result = (PyObject *)context; + return process_task_and_waiters(unwinder, key_addr, result); +} + +// Processor function for parsing tasks in sets +static int +process_task_parser( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *awaited_by = (PyObject *)context; + return parse_task(unwinder, key_addr, awaited_by); +} + +/* ============================================================================ + * MEMORY READING FUNCTIONS + * ============================================================================ */ + +#define DEFINE_MEMORY_READER(type_name, c_type, error_msg) \ +static inline int \ +read_##type_name(RemoteUnwinderObject *unwinder, uintptr_t address, c_type *result) \ +{ \ + int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(c_type), result); \ + if (res < 0) { \ + set_exception_cause(unwinder, PyExc_RuntimeError, error_msg); \ + return -1; \ + } \ + return 0; \ +} + +DEFINE_MEMORY_READER(ptr, uintptr_t, "Failed to read pointer from remote memory") +DEFINE_MEMORY_READER(Py_ssize_t, Py_ssize_t, "Failed to read Py_ssize_t from remote memory") +DEFINE_MEMORY_READER(char, char, "Failed to read char from remote memory") + static int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) { @@ -442,17 +582,6 @@ read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_ad return 0; } -static int -read_char(RemoteUnwinderObject *unwinder, uintptr_t address, char *result) -{ - int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(char), result); - if (res < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read char from remote memory"); - return -1; - } - return 0; -} - /* ============================================================================ * PYTHON OBJECT READING FUNCTIONS * ============================================================================ */ @@ -682,7 +811,7 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) } #elif defined(__linux__) // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_linux_map_for_section(handle, "AsyncioDebug", "python"); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); @@ -691,10 +820,10 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) } #elif defined(__APPLE__) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_map_for_section(handle, "AsyncioDebug", "libpython"); if (address == 0) { PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_map_for_section(handle, "AsyncioDebug", "python"); } if (address == 0) { // Error out: 'python' substring covers both executable and DLL @@ -799,39 +928,9 @@ parse_task_name( static int parse_task_awaited_by( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *awaited_by, - int recurse_task + PyObject *awaited_by ) { - // Read the entire TaskObj at once - char task_obj[SIZEOF_TASK_OBJ]; - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, - task_obj) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object in awaited_by parsing"); - return -1; - } - - uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); - - if ((void*)task_ab_addr == NULL) { - return 0; - } - - char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); - - if (awaited_by_is_a_set) { - if (parse_tasks_in_set(unwinder, task_ab_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse tasks in awaited_by set"); - return -1; - } - } else { - if (parse_task(unwinder, task_ab_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse single awaited_by task"); - return -1; - } - } - - return 0; + return process_task_awaited_by(unwinder, task_address, process_task_parser, awaited_by); } static int @@ -940,11 +1039,16 @@ parse_coro_chain( // Parse the previous frame using the gi_iframe from local copy uintptr_t prev_frame; uintptr_t gi_iframe_addr = coro_address + unwinder->debug_offsets.gen_object.gi_iframe; - if (parse_frame_object(unwinder, &name, gi_iframe_addr, &prev_frame) < 0) { + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); return -1; } + if (!name) { + return 0; + } + if (PyList_Append(render_to, name)) { Py_DECREF(name); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); @@ -962,8 +1066,7 @@ parse_coro_chain( static PyObject* create_task_result( RemoteUnwinderObject *unwinder, - uintptr_t task_address, - int recurse_task + uintptr_t task_address ) { PyObject* result = NULL; PyObject *call_stack = NULL; @@ -979,11 +1082,7 @@ create_task_result( } // Create task name/address for second tuple element - if (recurse_task) { - tn = parse_task_name(unwinder, task_address); - } else { - tn = PyLong_FromUnsignedLongLong(task_address); - } + tn = PyLong_FromUnsignedLongLong(task_address); if (tn == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name/address"); goto error; @@ -1036,8 +1135,7 @@ static int parse_task( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *render_to, - int recurse_task + PyObject *render_to ) { char is_task; PyObject* result = NULL; @@ -1053,7 +1151,7 @@ parse_task( } if (is_task) { - result = create_task_result(unwinder, task_address, recurse_task); + result = create_task_result(unwinder, task_address); if (!result) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); goto error; @@ -1080,11 +1178,11 @@ parse_task( PyStructSequence_SetItem(result, 0, empty_list); // This steals the reference PyStructSequence_SetItem(result, 1, task_name); // This steals the reference } - if (PyList_Append(render_to, result)) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task result to render list"); goto error; } + Py_DECREF(result); return 0; @@ -1094,234 +1192,335 @@ parse_task( } static int -process_set_entry( +process_single_task_node( RemoteUnwinderObject *unwinder, - uintptr_t table_ptr, - PyObject *awaited_by, - int recurse_task + uintptr_t task_addr, + PyObject **task_info, + PyObject *result ) { - uintptr_t key_addr; - if (read_py_ptr(unwinder, table_ptr, &key_addr)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); - return -1; + PyObject *tn = NULL; + PyObject *current_awaited_by = NULL; + PyObject *task_id = NULL; + PyObject *result_item = NULL; + PyObject *coroutine_stack = NULL; + + tn = parse_task_name(unwinder, task_addr); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); + goto error; } - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry reference count"); - return -1; - } + current_awaited_by = PyList_New(0); + if (current_awaited_by == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); + goto error; + } - if (ref_cnt) { - // if 'ref_cnt=0' it's a set dummy marker - if (parse_task(unwinder, key_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task in set entry"); - return -1; - } - return 1; // Successfully processed a valid entry - } + // Extract the coroutine stack for this task + coroutine_stack = PyList_New(0); + if (coroutine_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); + goto error; } - return 0; // Entry was NULL or dummy marker -} -static int -parse_tasks_in_set( - RemoteUnwinderObject *unwinder, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task -) { - char set_object[SIZEOF_SET_OBJ]; - int err = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - set_addr, - SIZEOF_SET_OBJ, - set_object); - if (err < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); - return -1; + if (parse_task(unwinder, task_addr, coroutine_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); + goto error; } - Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); - Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; // The set contains the `mask+1` element slots. - uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + task_id = PyLong_FromUnsignedLongLong(task_addr); + if (task_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); + goto error; + } - Py_ssize_t i = 0; - Py_ssize_t els = 0; - while (i < set_len && els < num_els) { - int result = process_set_entry(unwinder, table_ptr, awaited_by, recurse_task); + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result_item = PyStructSequence_New(state->TaskInfo_Type); + if (result_item == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); + goto error; + } - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process set entry"); - return -1; - } - if (result > 0) { - els++; - } + PyStructSequence_SetItem(result_item, 0, task_id); // steals ref + PyStructSequence_SetItem(result_item, 1, tn); // steals ref + PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref + PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref - table_ptr += sizeof(void*) * 2; - i++; + // References transferred to tuple + task_id = NULL; + tn = NULL; + coroutine_stack = NULL; + current_awaited_by = NULL; + + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + return -1; } + if (task_info != NULL) { + *task_info = result_item; + } + Py_DECREF(result_item); + + // Get back current_awaited_by reference for parse_task_awaited_by + current_awaited_by = PyStructSequence_GetItem(result_item, 3); + if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); + // No cleanup needed here since all references were transferred to result_item + // and result_item was already added to result list and decreffed + return -1; + } + return 0; + +error: + Py_XDECREF(tn); + Py_XDECREF(current_awaited_by); + Py_XDECREF(task_id); + Py_XDECREF(result_item); + Py_XDECREF(coroutine_stack); + return -1; } +// Thread processor for get_all_awaited_by +static int +process_thread_for_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + uintptr_t head_addr = thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + return append_awaited_by(unwinder, tid, head_addr, result); +} +// Generic function to process task awaited_by static int -setup_async_result_structure(RemoteUnwinderObject *unwinder, PyObject **result, PyObject **calls) -{ - *result = PyList_New(1); - if (*result == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create async result structure"); +process_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + set_entry_processor_func processor, + void *context +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; } - *calls = PyList_New(0); - if (*calls == NULL) { - Py_DECREF(*result); - *result = NULL; - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create calls list in async result"); - return -1; + uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); + if ((void*)task_ab_addr == NULL) { + return 0; // No tasks waiting for this one } - if (PyList_SetItem(*result, 0, *calls)) { /* steals ref to 'calls' */ - Py_DECREF(*calls); - Py_DECREF(*result); - *result = NULL; - *calls = NULL; - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to set calls list in async result"); - return -1; - } + char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); - return 0; + if (awaited_by_is_a_set) { + return iterate_set_entries(unwinder, task_ab_addr, processor, context); + } else { + // Single task waiting + return processor(unwinder, task_ab_addr, context); + } } static int -add_task_info_to_result( +process_running_task_chain( RemoteUnwinderObject *unwinder, - PyObject *result, - uintptr_t running_task_addr + uintptr_t running_task_addr, + uintptr_t thread_state_addr, + PyObject *result ) { - PyObject *tn = parse_task_name(unwinder, running_task_addr); - if (tn == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name for result"); + uintptr_t running_task_code_obj = 0; + if(get_task_code_object(unwinder, running_task_addr, &running_task_code_obj) < 0) { + return -1; + } + + // First, add this task to the result + PyObject *task_info = NULL; + if (process_single_task_node(unwinder, running_task_addr, &task_info, result) < 0) { return -1; } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task name to result"); + // Get the chain from the current frame to this task + PyObject *coro_chain = PyStructSequence_GET_ITEM(task_info, 2); + assert(coro_chain != NULL); + if (PyList_GET_SIZE(coro_chain) != 1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Coro chain is not a single item"); return -1; } - Py_DECREF(tn); + PyObject *coro_info = PyList_GET_ITEM(coro_chain, 0); + assert(coro_info != NULL); + PyObject *frame_chain = PyStructSequence_GET_ITEM(coro_info, 0); + assert(frame_chain != NULL); - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list for result"); + // Clear the coro_chain + if (PyList_Clear(frame_chain) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to clear coroutine chain"); return -1; } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by to result"); + // Add the chain from the current frame to this task + if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_code_obj) < 0) { return -1; } - Py_DECREF(awaited_by); - if (parse_task_awaited_by( - unwinder, running_task_addr, awaited_by, 1) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by for result"); + // Now find all tasks that are waiting for this task and process them + if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, result) < 0) { return -1; } return 0; } +// Thread processor for get_async_stack_trace static int -process_single_task_node( +process_thread_for_async_stack_trace( RemoteUnwinderObject *unwinder, - uintptr_t task_addr, - PyObject *result + uintptr_t thread_state_addr, + unsigned long tid, + void *context ) { - PyObject *tn = NULL; - PyObject *current_awaited_by = NULL; - PyObject *task_id = NULL; - PyObject *result_item = NULL; - PyObject *coroutine_stack = NULL; + PyObject *result = (PyObject *)context; - tn = parse_task_name(unwinder, task_addr); - if (tn == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); - goto error; + // Find running task in this thread + uintptr_t running_task_addr; + if (find_running_task_in_thread(unwinder, thread_state_addr, &running_task_addr) < 0) { + return 0; } - current_awaited_by = PyList_New(0); - if (current_awaited_by == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); - goto error; + // If we found a running task, process it and its waiters + if ((void*)running_task_addr != NULL) { + PyObject *task_list = PyList_New(0); + if (task_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create task list for thread"); + return -1; + } + + if (process_running_task_chain(unwinder, running_task_addr, thread_state_addr, task_list) < 0) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process running task chain"); + return -1; + } + + // Create AwaitedInfo structure for this thread + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + return -1; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *awaited_info = PyStructSequence_New(state->AwaitedInfo_Type); + if (awaited_info == NULL) { + Py_DECREF(tid_py); + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } + + PyStructSequence_SetItem(awaited_info, 0, tid_py); // steals ref + PyStructSequence_SetItem(awaited_info, 1, task_list); // steals ref + + if (PyList_Append(result, awaited_info)) { + Py_DECREF(awaited_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append AwaitedInfo to result"); + return -1; + } + Py_DECREF(awaited_info); } - // Extract the coroutine stack for this task - coroutine_stack = PyList_New(0); - if (coroutine_stack == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); - goto error; + return 0; +} + +static int +process_task_and_waiters( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject *result +) { + // First, add this task to the result + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { + return -1; } - if (parse_task(unwinder, task_addr, coroutine_stack, 0) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); - goto error; + // Now find all tasks that are waiting for this task and process them + return process_task_awaited_by(unwinder, task_addr, process_waiter_task, result); +} + +static int +find_running_task_in_thread( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + uintptr_t address_of_running_loop; + int bytes_read = read_py_ptr( + unwinder, + thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); + return -1; } - task_id = PyLong_FromUnsignedLongLong(task_addr); - if (task_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); - goto error; + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - result_item = PyStructSequence_New(state->TaskInfo_Type); - if (result_item == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); - goto error; + int err = read_ptr( + unwinder, + thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); + return -1; } - PyStructSequence_SetItem(result_item, 0, task_id); // steals ref - PyStructSequence_SetItem(result_item, 1, tn); // steals ref - PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref - PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref + return 0; +} - // References transferred to tuple - task_id = NULL; - tn = NULL; - coroutine_stack = NULL; - current_awaited_by = NULL; +static int +get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr) { + uintptr_t running_coro_addr = 0; - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + if(read_py_ptr( + unwinder, + task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, + &running_coro_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); return -1; } - Py_DECREF(result_item); - // Get back current_awaited_by reference for parse_task_awaited_by - current_awaited_by = PyStructSequence_GetItem(result_item, 3); - if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by, 0) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); - // No cleanup needed here since all references were transferred to result_item - // and result_item was already added to result list and decreffed + if (running_coro_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); return -1; } - return 0; + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + if (read_py_ptr( + unwinder, + running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); + return -1; + } -error: - Py_XDECREF(tn); - Py_XDECREF(current_awaited_by); - Py_XDECREF(task_id); - Py_XDECREF(result_item); - Py_XDECREF(coroutine_stack); - return -1; + if (*code_obj_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); + return -1; + } + + return 0; } /* ============================================================================ @@ -1904,45 +2103,13 @@ populate_initial_state_data( return 0; } + static int find_running_frame( RemoteUnwinderObject *unwinder, - uintptr_t runtime_start_address, + uintptr_t address_of_thread, uintptr_t *frame ) { - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state for running frame"); - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL in running frame search"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_main, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread address for running frame"); - return -1; - } - - // No Python frames are available for us (can happen at tear-down). if ((void*)address_of_thread != NULL) { int err = read_ptr( unwinder, @@ -1959,133 +2126,6 @@ find_running_frame( return 0; } -static int -find_running_task( - RemoteUnwinderObject *unwinder, - uintptr_t *running_task_addr -) { - *running_task_addr = (uintptr_t)NULL; - - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - unwinder->runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state for running task"); - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL in running task search"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_head, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread head for running task"); - return -1; - } - - uintptr_t address_of_running_loop; - // No Python frames are available for us (can happen at tear-down). - if ((void*)address_of_thread == NULL) { - return 0; - } - - bytes_read = read_py_ptr( - unwinder, - address_of_thread - + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, - &address_of_running_loop); - if (bytes_read == -1) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); - return -1; - } - - // no asyncio loop is now running - if ((void*)address_of_running_loop == NULL) { - return 0; - } - - int err = read_ptr( - unwinder, - address_of_thread - + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, - running_task_addr); - if (err) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); - return -1; - } - - return 0; -} - -static int -find_running_task_and_coro( - RemoteUnwinderObject *unwinder, - uintptr_t *running_task_addr, - uintptr_t *running_coro_addr, - uintptr_t *running_task_code_obj -) { - *running_task_addr = (uintptr_t)NULL; - if (find_running_task( - unwinder, running_task_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task search failed"); - return -1; - } - - if ((void*)*running_task_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No running task found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task address is NULL"); - return -1; - } - - if (read_py_ptr( - unwinder, - *running_task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, - running_coro_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); - return -1; - } - - if ((void*)*running_coro_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); - return -1; - } - - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - if (read_py_ptr( - unwinder, - *running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, - running_task_code_obj) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); - return -1; - } - - if ((void*)*running_task_code_obj == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); - return -1; - } - - return 0; -} - - /* ============================================================================ * FRAME PARSING FUNCTIONS * ============================================================================ */ @@ -2122,9 +2162,11 @@ parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, + uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { char frame[SIZEOF_INTERP_FRAME]; + *address_of_code_object = 0; Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, @@ -2154,77 +2196,32 @@ parse_frame_object( } #endif - return parse_code_object(unwinder, result, code_object,instruction_pointer, previous_frame, tlbc_index); -} - -static int -parse_async_frame_object( - RemoteUnwinderObject *unwinder, - PyObject** result, - uintptr_t address, - uintptr_t* previous_frame, - uintptr_t* code_object -) { - char frame[SIZEOF_INTERP_FRAME]; - - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read async frame"); - return -1; - } - - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); - *code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); - int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, *code_object); - if (frame_valid != 1) { - return frame_valid; - } - - uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); - - // Get tlbc_index for free threading builds - int32_t tlbc_index = 0; -#ifdef Py_GIL_DISABLED - if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { - tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); - } -#endif - - if (parse_code_object( - unwinder, result, *code_object, instruction_pointer, previous_frame, tlbc_index)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse code object in async frame"); - return -1; - } - - return 1; + *address_of_code_object = code_object; + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); } static int -parse_async_frame_chain( + parse_async_frame_chain( RemoteUnwinderObject *unwinder, PyObject *calls, + uintptr_t address_of_thread, uintptr_t running_task_code_obj ) { uintptr_t address_of_current_frame; - if (find_running_frame(unwinder, unwinder->runtime_start_address, &address_of_current_frame) < 0) { + if (find_running_frame(unwinder, address_of_thread, &address_of_current_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Running frame search failed in async chain"); return -1; } - uintptr_t address_of_code_object; while ((void*)address_of_current_frame != NULL) { PyObject* frame_info = NULL; - int res = parse_async_frame_object( + uintptr_t address_of_code_object; + int res = parse_frame_object( unwinder, &frame_info, address_of_current_frame, - &address_of_current_frame, - &address_of_code_object + &address_of_code_object, + &address_of_current_frame ); if (res < 0) { @@ -2290,7 +2287,7 @@ append_awaited_by_for_thread( uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) - unwinder->async_debug_offsets.asyncio_task_object.task_node; - if (process_single_task_node(unwinder, task_addr, result) < 0) { + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); return -1; } @@ -2385,7 +2382,8 @@ process_frame_chain( // Try chunks first, fallback to direct memory read if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { PyErr_Clear(); - if (parse_frame_object(unwinder, &frame, frame_addr, &next_frame_addr) < 0) { + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } @@ -2762,6 +2760,7 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } exit: + _Py_RemoteDebug_ClearCache(&self->handle); return result; } @@ -2825,53 +2824,12 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s goto result_err; } - uintptr_t thread_state_addr; - unsigned long tid = 0; - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - self->interpreter_addr - + self->debug_offsets.interpreter_state.threads_main, - sizeof(void*), - &thread_state_addr)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read main thread state in get_all_awaited_by"); + // Process all threads + if (iterate_threads(self, process_thread_for_awaited_by, result) < 0) { goto result_err; } - uintptr_t head_addr; - while (thread_state_addr != 0) { - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - thread_state_addr - + self->debug_offsets.thread_state.native_thread_id, - sizeof(tid), - &tid)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read thread ID in get_all_awaited_by"); - goto result_err; - } - - head_addr = thread_state_addr - + self->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; - - if (append_awaited_by(self, tid, head_addr, result)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to append awaited_by for thread in get_all_awaited_by"); - goto result_err; - } - - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - thread_state_addr + self->debug_offsets.thread_state.next, - sizeof(void*), - &thread_state_addr)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read next thread state in get_all_awaited_by"); - goto result_err; - } - } - - head_addr = self->interpreter_addr + uintptr_t head_addr = self->interpreter_addr + self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; // On top of a per-thread task lists used by default by asyncio to avoid @@ -2885,9 +2843,11 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s goto result_err; } + _Py_RemoteDebug_ClearCache(&self->handle); return result; result_err: + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2896,32 +2856,50 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s @critical_section _remote_debugging.RemoteUnwinder.get_async_stack_trace -Returns information about the currently running async task and its stack trace. +Get the currently running async tasks and their dependency graphs from the remote process. -Returns a tuple of (task_info, stack_frames) where: -- task_info is a tuple of (task_id, task_name) identifying the task -- stack_frames is a list of tuples (function_name, filename, line_number) representing - the Python stack frames for the task, ordered from most recent to oldest +This returns information about running tasks and all tasks that are waiting for them, +forming a complete dependency graph for each thread's active task. -Example: - ((4345585712, 'Task-1'), [ - ('run_echo_server', 'server.py', 127), - ('serve_forever', 'server.py', 45), - ('main', 'app.py', 23) - ]) +For each thread with a running task, returns the running task plus all tasks that +transitively depend on it (tasks waiting for the running task, tasks waiting for +those tasks, etc.). + +Returns a list of per-thread results, where each thread result contains: +- Thread ID +- List of task information for the running task and all its waiters + +Each task info contains: +- Task ID (memory address) +- Task name +- Call stack frames: List of (func_name, filename, lineno) +- List of tasks waiting for this task (recursive structure) Raises: RuntimeError: If AsyncioDebug section is not available in the target process - RuntimeError: If there is an error copying memory from the target process - OSError: If there is an error accessing the target process - PermissionError: If access to the target process is denied - UnicodeDecodeError: If there is an error decoding strings from the target process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output (similar structure to get_all_awaited_by but only for running tasks): +[ + # Thread 140234 results + (140234, [ + # Running task and its complete waiter dependency graph + (4345585712, 'main_task', + [("run_server", "server.py", 127), ("main", "app.py", 23)], + [ + # Tasks waiting for main_task + (4345585800, 'worker_1', [...], [...]), + (4345585900, 'worker_2', [...], [...]) + ]) + ]) +] [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=11b7150c59d4c60f]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=8744b47c9ec2220a]*/ { if (!self->async_debug_offsets_available) { PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); @@ -2929,34 +2907,21 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject return NULL; } - PyObject *result = NULL; - PyObject *calls = NULL; - - if (setup_async_result_structure(self, &result, &calls) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to setup async result structure"); - goto cleanup; - } - - uintptr_t running_task_addr, running_coro_addr, running_task_code_obj; - if (find_running_task_and_coro(self, &running_task_addr, - &running_coro_addr, &running_task_code_obj) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to find running task and coro"); - goto cleanup; - } - - if (parse_async_frame_chain(self, calls, running_task_code_obj) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to parse async frame chain"); - goto cleanup; + PyObject *result = PyList_New(0); + if (result == NULL) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create result list in get_async_stack_trace"); + return NULL; } - if (add_task_info_to_result(self, result, running_task_addr) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to add task info to result"); - goto cleanup; + // Process all threads + if (iterate_threads(self, process_thread_for_async_stack_trace, result) < 0) { + goto result_err; } + _Py_RemoteDebug_ClearCache(&self->handle); return result; - -cleanup: +result_err: + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2982,6 +2947,7 @@ RemoteUnwinder_dealloc(PyObject *op) } #endif if (self->handle.pid != 0) { + _Py_RemoteDebug_ClearCache(&self->handle); _Py_RemoteDebug_CleanupProcHandle(&self->handle); } PyObject_Del(self); diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 35d090e3ca2dce..aafefbf316e03d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,6 +4,7 @@ #include "blob.h" #include "util.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -56,9 +57,7 @@ blob_dealloc(PyObject *op) close_blob(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 7943bfcca3679d..0c3f43d0e50b43 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -31,6 +31,7 @@ #include "util.h" #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef enum { TYPE_LONG, @@ -185,9 +186,7 @@ cursor_dealloc(PyObject *op) pysqlite_Cursor *self = _pysqlite_Cursor_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 602d0ab8588f62..e8943920043906 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -44,6 +44,7 @@ static const char copyright[] = #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_unicodeobject.h" // _PyUnicode_Copy +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "sre.h" // SRE_CODE @@ -736,10 +737,7 @@ pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - PatternObject *obj = _PatternObject_CAST(self); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, _PatternObject_CAST(self)->weakreflist); (void)pattern_clear(self); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 014e624f6c2f00..24c243e330d4bf 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -75,6 +75,33 @@ #endif +#ifdef BIO_get_ktls_send +# ifdef MS_WINDOWS +typedef long long Py_off_t; +# else +typedef off_t Py_off_t; +# endif + +static int +Py_off_t_converter(PyObject *arg, void *addr) +{ +#ifdef HAVE_LARGEFILE_SUPPORT + *((Py_off_t *)addr) = PyLong_AsLongLong(arg); +#else + *((Py_off_t *)addr) = PyLong_AsLong(arg); +#endif + return PyErr_Occurred() ? 0 : 1; +} + +/*[python input] + +class Py_off_t_converter(CConverter): + type = 'Py_off_t' + converter = 'Py_off_t_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=3fd9ca8ca6f0cbb8]*/ +#endif /* BIO_get_ktls_send */ struct py_ssl_error_code { const char *mnemonic; @@ -2442,6 +2469,184 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) return rc == 0 ? SOCKET_HAS_TIMED_OUT : SOCKET_OPERATION_OK; } +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_send + +Check if the Kernel TLS data-path is used for sending. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self) +/*[clinic end generated code: output=f9d95fbefceb5068 input=8d1ce4a131190e6b]*/ +{ +#ifdef BIO_get_ktls_send + int uses = BIO_get_ktls_send(SSL_get_wbio(self->ssl)); + // BIO_get_ktls_send() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_recv + +Check if the Kernel TLS data-path is used for receiving. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) +/*[clinic end generated code: output=ce38b00317a1f681 input=a13778a924fc7d44]*/ +{ +#ifdef BIO_get_ktls_recv + int uses = BIO_get_ktls_recv(SSL_get_rbio(self->ssl)); + // BIO_get_ktls_recv() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +#ifdef BIO_get_ktls_send +/*[clinic input] +@critical_section +_ssl._SSLSocket.sendfile + fd: int + offset: Py_off_t + size: size_t + flags: int = 0 + / + +Write size bytes from offset in the file descriptor fd to the SSL connection. + +This method uses the zero-copy technique and returns the number of bytes +written. It should be called only when Kernel TLS is used for sending data in +the connection. + +The meaning of flags is platform dependent. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags) +/*[clinic end generated code: output=0c6815b0719ca8d5 input=dfc1b162bb020de1]*/ +{ + Py_ssize_t retval; + int sockstate; + _PySSLError err; + PySocketSockObject *sock = GET_SOCKET(self); + PyTime_t timeout, deadline = 0; + int has_timeout; + + if (sock != NULL) { + if ((PyObject *)sock == Py_None) { + _setSSLError(get_state_sock(self), + "Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + /* just in case the blocking state of the socket has been changed */ + int nonblocking = (sock->sock_timeout >= 0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } + + timeout = GET_SOCKET_TIMEOUT(sock); + has_timeout = (timeout > 0); + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } + + sockstate = PySSL_select(sock, 1, timeout); + switch (sockstate) { + case SOCKET_HAS_TIMED_OUT: + PyErr_SetString(PyExc_TimeoutError, + "The write operation timed out"); + goto error; + case SOCKET_HAS_BEEN_CLOSED: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + case SOCKET_TOO_LARGE_FOR_SELECT: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket too large for select()."); + goto error; + } + + do { + PySSL_BEGIN_ALLOW_THREADS + retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); + err = _PySSL_errno(retval < 0, self->ssl, (int)retval); + PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) { + goto error; + } + + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } + + switch (err.ssl) { + case SSL_ERROR_WANT_READ: + sockstate = PySSL_select(sock, 0, timeout); + break; + case SSL_ERROR_WANT_WRITE: + sockstate = PySSL_select(sock, 1, timeout); + break; + default: + sockstate = SOCKET_OPERATION_OK; + break; + } + + if (sockstate == SOCKET_HAS_TIMED_OUT) { + PyErr_SetString(PyExc_TimeoutError, + "The sendfile operation timed out"); + goto error; + } + else if (sockstate == SOCKET_HAS_BEEN_CLOSED) { + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + } + else if (sockstate == SOCKET_IS_NONBLOCKING) { + break; + } + } while (err.ssl == SSL_ERROR_WANT_READ + || err.ssl == SSL_ERROR_WANT_WRITE); + + if (err.ssl == SSL_ERROR_SSL + && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED) + { + /* OpenSSL fails to return SSL_ERROR_SYSCALL if an error + * happens in sendfile(), and returns SSL_ERROR_SSL with + * SSL_R_UNINITIALIZED reason instead. */ + _setSSLError(get_state_sock(self), + "Some I/O error occurred in sendfile()", + PY_SSL_ERROR_SYSCALL, __FILE__, __LINE__); + goto error; + } + Py_XDECREF(sock); + if (retval < 0) { + return PySSL_SetError(self, __FILE__, __LINE__); + } + if (PySSL_ChainExceptions(self) < 0) { + return NULL; + } + return PyLong_FromSize_t(retval); +error: + Py_XDECREF(sock); + (void)PySSL_ChainExceptions(self); + return NULL; +} +#endif /* BIO_get_ktls_send */ + /*[clinic input] @critical_section _ssl._SSLSocket.write @@ -3017,6 +3222,9 @@ static PyGetSetDef ssl_getsetlist[] = { static PyMethodDef PySSLMethods[] = { _SSL__SSLSOCKET_DO_HANDSHAKE_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF + _SSL__SSLSOCKET_SENDFILE_METHODDEF _SSL__SSLSOCKET_WRITE_METHODDEF _SSL__SSLSOCKET_READ_METHODDEF _SSL__SSLSOCKET_PENDING_METHODDEF diff --git a/Modules/_struct.c b/Modules/_struct.c index c36079f1eb8886..3fad35a8c94ee2 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -11,6 +11,7 @@ #include "pycore_bytesobject.h" // _PyBytesWriter #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() @@ -1794,9 +1795,7 @@ s_dealloc(PyObject *op) PyStructObject *s = PyStructObject_CAST(op); PyTypeObject *tp = Py_TYPE(s); PyObject_GC_UnTrack(s); - if (s->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, s->weakreflist); if (s->s_codes != NULL) { PyMem_Free(s->s_codes); } diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 7fc4d61db29469..d2e61e9d6acf24 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1855,8 +1855,7 @@ ndarray_subscript(PyObject *op, PyObject *key) type_error: PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", - Py_TYPE(key)->tp_name); + "cannot index memory using \"%T\"", key); err_occurred: Py_DECREF(nd); return NULL; diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index d4045afd515909..c1f769456accef 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -178,6 +178,42 @@ sequence_fast_get_item(PyObject *self, PyObject *args) } +static PyObject * +object_setattr_null_exc(PyObject *self, PyObject *args) +{ + PyObject *obj, *name, *exc; + if (!PyArg_ParseTuple(args, "OOO", &obj, &name, &exc)) { + return NULL; + } + + PyErr_SetObject((PyObject*)Py_TYPE(exc), exc); + if (PyObject_SetAttr(obj, name, NULL) < 0) { + return NULL; + } + assert(PyErr_Occurred()); + return NULL; +} + + +static PyObject * +object_setattrstring_null_exc(PyObject *self, PyObject *args) +{ + PyObject *obj, *exc; + const char *name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &name, &size, &exc)) { + return NULL; + } + + PyErr_SetObject((PyObject*)Py_TYPE(exc), exc); + if (PyObject_SetAttrString(obj, name, NULL) < 0) { + return NULL; + } + assert(PyErr_Occurred()); + return NULL; +} + + static PyMethodDef test_methods[] = { {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, @@ -191,6 +227,8 @@ static PyMethodDef test_methods[] = { {"sequence_fast_get_size", sequence_fast_get_size, METH_O}, {"sequence_fast_get_item", sequence_fast_get_item, METH_VARARGS}, + {"object_setattr_null_exc", object_setattr_null_exc, METH_VARARGS}, + {"object_setattrstring_null_exc", object_setattrstring_null_exc, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c index 08a2055c51bc0d..e041943492d61f 100644 --- a/Modules/_testcapi/monitoring.c +++ b/Modules/_testcapi/monitoring.c @@ -109,7 +109,7 @@ static PyTypeObject PyCodeLike_Type = { }; #define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ - PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %T", v); \ return NULL; \ } diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 464cf5c3125012..4ca6ff587b95cc 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -5,8 +5,7 @@ static int pytime_from_nanoseconds(PyTime_t *tp, PyObject *obj) { if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_TypeError, "expect int, got %s", - Py_TYPE(obj)->tp_name); + PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 03aaacb328e0b6..f89dcb6c4cf03c 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -179,14 +179,14 @@ _testcapi_VectorCallClass_set_vectorcall_impl(PyObject *self, if (!PyObject_TypeCheck(self, type)) { return PyErr_Format( PyExc_TypeError, - "expected %s instance", - PyType_GetName(type)); + "expected %N instance", + type); } if (!type->tp_vectorcall_offset) { return PyErr_Format( PyExc_TypeError, - "type %s has no vectorcall offset", - PyType_GetName(type)); + "type %N has no vectorcall offset", + type); } *(vectorcallfunc*)((char*)self + type->tp_vectorcall_offset) = ( VectorCallClass_vectorcall); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 71fffedee146fa..d0c0b45c20cb36 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -515,8 +515,7 @@ test_thread_state(PyObject *self, PyObject *args) return NULL; if (!PyCallable_Check(fn)) { - PyErr_Format(PyExc_TypeError, "'%s' object is not callable", - Py_TYPE(fn)->tp_name); + PyErr_Format(PyExc_TypeError, "'%T' object is not callable", fn); return NULL; } @@ -2419,6 +2418,16 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) Py_BEGIN_CRITICAL_SECTION2(module, module); Py_END_CRITICAL_SECTION2(); +#ifdef Py_GIL_DISABLED + // avoid unused variable compiler warning on GIL-enabled build + PyMutex mut = {0}; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&mut); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2_MUTEX(&mut, &mut); + Py_END_CRITICAL_SECTION2(); +#endif + Py_RETURN_NONE; } diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 7e4ea0901fe726..3e903b6d87d89f 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1734,14 +1734,14 @@ output impl_definition block class _testclinic.DeprStarNew "PyObject *" "PyObject" @classmethod _testclinic.DeprStarNew.__new__ as depr_star_new - * [from 3.37] + * [from 3.14] a: object = None The deprecation message should use the class name instead of __new__. [clinic start generated code]*/ static PyObject * depr_star_new_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=bdbb36244f90cf46 input=df8930826b302c3a]*/ +/*[clinic end generated code: output=bdbb36244f90cf46 input=fdd640db964b4dc1]*/ { return type->tp_alloc(type, 0); } @@ -1775,14 +1775,14 @@ static PyTypeObject DeprStarNew = { /*[clinic input] class _testclinic.DeprStarInit "PyObject *" "PyObject" _testclinic.DeprStarInit.__init__ as depr_star_init - * [from 3.37] + * [from 3.14] a: object = None The deprecation message should use the class name instead of __init__. [clinic start generated code]*/ static int depr_star_init_impl(PyObject *self, PyObject *a) -/*[clinic end generated code: output=8d27b43c286d3ecc input=07a5c35e04f526c5]*/ +/*[clinic end generated code: output=8d27b43c286d3ecc input=5575b77229d5e2be]*/ { return 0; } @@ -1818,7 +1818,7 @@ static PyTypeObject DeprStarInit = { class _testclinic.DeprStarInitNoInline "PyObject *" "PyObject" _testclinic.DeprStarInitNoInline.__init__ as depr_star_init_noinline a: object - * [from 3.37] + * [from 3.14] b: object c: object = None * @@ -1829,7 +1829,7 @@ _testclinic.DeprStarInitNoInline.__init__ as depr_star_init_noinline static int depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length) -/*[clinic end generated code: output=9b31fc167f1bf9f7 input=45171504f009a391]*/ +/*[clinic end generated code: output=9b31fc167f1bf9f7 input=5a887543122bca48]*/ { return 0; } @@ -1849,13 +1849,13 @@ class _testclinic.DeprKwdNew "PyObject *" "PyObject" @classmethod _testclinic.DeprKwdNew.__new__ as depr_kwd_new a: object = None - / [from 3.37] + / [from 3.14] The deprecation message should use the class name instead of __new__. [clinic start generated code]*/ static PyObject * depr_kwd_new_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=618d07afc5616149 input=1bfb1b86f56ad2e6]*/ +/*[clinic end generated code: output=618d07afc5616149 input=6c7d13c471013c10]*/ { return type->tp_alloc(type, 0); } @@ -1873,13 +1873,13 @@ static PyTypeObject DeprKwdNew = { class _testclinic.DeprKwdInit "PyObject *" "PyObject" _testclinic.DeprKwdInit.__init__ as depr_kwd_init a: object = None - / [from 3.37] + / [from 3.14] The deprecation message should use the class name instead of __init__. [clinic start generated code]*/ static int depr_kwd_init_impl(PyObject *self, PyObject *a) -/*[clinic end generated code: output=6e02eb724a85d840 input=6f4daaa912ec24b2]*/ +/*[clinic end generated code: output=6e02eb724a85d840 input=b9bf3c20f012d539]*/ { return 0; } @@ -1901,7 +1901,7 @@ _testclinic.DeprKwdInitNoInline.__init__ as depr_kwd_init_noinline / b: object c: object = None - / [from 3.37] + / [from 3.14] # Force to use _PyArg_ParseTupleAndKeywordsFast. d: str(accept={str, robuffer}, zeroes=True) = '' [clinic start generated code]*/ @@ -1909,7 +1909,7 @@ _testclinic.DeprKwdInitNoInline.__init__ as depr_kwd_init_noinline static int depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length) -/*[clinic end generated code: output=27759d70ddd25873 input=a022ad17f4b6008c]*/ +/*[clinic end generated code: output=27759d70ddd25873 input=c19d982c8c70a930]*/ { return 0; } @@ -1926,13 +1926,13 @@ static PyTypeObject DeprKwdInitNoInline = { /*[clinic input] depr_star_pos0_len1 - * [from 3.37] + * [from 3.14] a: object [clinic start generated code]*/ static PyObject * depr_star_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=e1c6c2b423129499 input=c8f49d8c6165ab6c]*/ +/*[clinic end generated code: output=e1c6c2b423129499 input=089b9aee25381b69]*/ { Py_RETURN_NONE; } @@ -1940,14 +1940,14 @@ depr_star_pos0_len1_impl(PyObject *module, PyObject *a) /*[clinic input] depr_star_pos0_len2 - * [from 3.37] + * [from 3.14] a: object b: object [clinic start generated code]*/ static PyObject * depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=96df9be39859c7e4 input=aca96f36892eda25]*/ +/*[clinic end generated code: output=96df9be39859c7e4 input=65c83a32e01495c6]*/ { Py_RETURN_NONE; } @@ -1955,7 +1955,7 @@ depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) /*[clinic input] depr_star_pos0_len3_with_kwd - * [from 3.37] + * [from 3.14] a: object b: object c: object @@ -1966,7 +1966,7 @@ depr_star_pos0_len3_with_kwd static PyObject * depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=7f2531eda837052f input=5602f0bced3d5094]*/ +/*[clinic end generated code: output=7f2531eda837052f input=b33f620f57d9270f]*/ { Py_RETURN_NONE; } @@ -1975,13 +1975,13 @@ depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic input] depr_star_pos1_len1_opt a: object - * [from 3.37] + * [from 3.14] b: object = None [clinic start generated code]*/ static PyObject * depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=b5b4e326ee3b216f input=070817da1d6ccf49]*/ +/*[clinic end generated code: output=b5b4e326ee3b216f input=4a4b8ff72eae9ff7]*/ { Py_RETURN_NONE; } @@ -1990,13 +1990,13 @@ depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b) /*[clinic input] depr_star_pos1_len1 a: object - * [from 3.37] + * [from 3.14] b: object [clinic start generated code]*/ static PyObject * depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=eab92e37d5b0a480 input=2e3a30c71edd0f30]*/ +/*[clinic end generated code: output=eab92e37d5b0a480 input=1e7787a9fe5f62a0]*/ { Py_RETURN_NONE; } @@ -2005,7 +2005,7 @@ depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) /*[clinic input] depr_star_pos1_len2_with_kwd a: object - * [from 3.37] + * [from 3.14] b: object c: object * @@ -2015,7 +2015,7 @@ depr_star_pos1_len2_with_kwd static PyObject * depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=3bccab672b7cfbb8 input=48492b028a4f281c]*/ +/*[clinic end generated code: output=3bccab672b7cfbb8 input=6bc7bd742fa8be15]*/ { Py_RETURN_NONE; } @@ -2025,14 +2025,14 @@ depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, depr_star_pos2_len1 a: object b: object - * [from 3.37] + * [from 3.14] c: object [clinic start generated code]*/ static PyObject * depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=20f5b230e9beeb70 input=80ee46e15cd14cf3]*/ +/*[clinic end generated code: output=20f5b230e9beeb70 input=5fc3e1790dec00d5]*/ { Py_RETURN_NONE; } @@ -2042,7 +2042,7 @@ depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, depr_star_pos2_len2 a: object b: object - * [from 3.37] + * [from 3.14] c: object d: object [clinic start generated code]*/ @@ -2050,7 +2050,7 @@ depr_star_pos2_len2 static PyObject * depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=9f90ed8fbce27d7a input=ac57914cf40a011b]*/ +/*[clinic end generated code: output=9f90ed8fbce27d7a input=9cc8003b89d38779]*/ { Py_RETURN_NONE; } @@ -2060,7 +2060,7 @@ depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, depr_star_pos2_len2_with_kwd a: object b: object - * [from 3.37] + * [from 3.14] c: object d: object * @@ -2070,7 +2070,7 @@ depr_star_pos2_len2_with_kwd static PyObject * depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=05432c4f20527215 input=98f25e33c01285a3]*/ +/*[clinic end generated code: output=05432c4f20527215 input=831832d90534da91]*/ { Py_RETURN_NONE; } @@ -2079,7 +2079,7 @@ depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic input] depr_star_noinline a: object - * [from 3.37] + * [from 3.14] b: object c: object = None * @@ -2090,7 +2090,7 @@ depr_star_noinline static PyObject * depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length) -/*[clinic end generated code: output=cc27dacf5c2754af input=a829784557a42349]*/ +/*[clinic end generated code: output=cc27dacf5c2754af input=d36cc862a2daef98]*/ { Py_RETURN_NONE; } @@ -2099,12 +2099,12 @@ depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, /*[clinic input] depr_star_multi a: object - * [from 3.39] + * [from 3.16] b: object - * [from 3.38] + * [from 3.15] c: object d: object - * [from 3.37] + * [from 3.14] e: object f: object g: object @@ -2116,7 +2116,7 @@ static PyObject * depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g, PyObject *h) -/*[clinic end generated code: output=77681653f4202068 input=6798940a18b2e62b]*/ +/*[clinic end generated code: output=77681653f4202068 input=3ebd05d888a957ea]*/ { Py_RETURN_NONE; } @@ -2127,12 +2127,12 @@ depr_kwd_required_1 a: object / b: object - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=1d8ab19ea78418af input=37853d8548d42df5]*/ +/*[clinic end generated code: output=1d8ab19ea78418af input=53f2c398b828462d]*/ { Py_RETURN_NONE; } @@ -2144,13 +2144,13 @@ depr_kwd_required_2 / b: object c: object - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=44a89cb82509ddde input=0e1faedd8ec248b1]*/ +/*[clinic end generated code: output=44a89cb82509ddde input=a2b0ef37de8a01a7]*/ { Py_RETURN_NONE; } @@ -2161,12 +2161,12 @@ depr_kwd_optional_1 a: object / b: object = None - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=a8a3d67efcc7b058 input=adb240416511b30a]*/ +/*[clinic end generated code: output=a8a3d67efcc7b058 input=e416981eb78c3053]*/ { Py_RETURN_NONE; } @@ -2178,13 +2178,13 @@ depr_kwd_optional_2 / b: object = None c: object = None - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=aa2d967f26fdb9f6 input=fb8a82d0087f8732]*/ +/*[clinic end generated code: output=aa2d967f26fdb9f6 input=cae3afb783bfc855]*/ { Py_RETURN_NONE; } @@ -2195,13 +2195,13 @@ depr_kwd_optional_3 a: object = None b: object = None c: object = None - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=a26025bf6118fd07 input=e54bbbacd8624fa7]*/ +/*[clinic end generated code: output=a26025bf6118fd07 input=c9183b2f9ccaf992]*/ { Py_RETURN_NONE; } @@ -2213,13 +2213,13 @@ depr_kwd_required_optional / b: object c: object = None - / [from 3.37] + / [from 3.14] [clinic start generated code]*/ static PyObject * depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=e53a8b7a250d8ffc input=285105a1ffb784b5]*/ +/*[clinic end generated code: output=e53a8b7a250d8ffc input=23237a046f8388f5]*/ { Py_RETURN_NONE; } @@ -2231,7 +2231,7 @@ depr_kwd_noinline / b: object c: object = None - / [from 3.37] + / [from 3.14] # Force to use _PyArg_ParseStackAndKeywords. d: str(accept={str, robuffer}, zeroes=True) = '' [clinic start generated code]*/ @@ -2239,7 +2239,7 @@ depr_kwd_noinline static PyObject * depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length) -/*[clinic end generated code: output=f59da8113f2bad7c input=3773d1bdc1f82298]*/ +/*[clinic end generated code: output=f59da8113f2bad7c input=1d6db65bebb069d7]*/ { Py_RETURN_NONE; } @@ -2250,14 +2250,14 @@ depr_kwd_multi a: object / b: object - / [from 3.37] + / [from 3.14] c: object d: object - / [from 3.38] + / [from 3.15] e: object f: object g: object - / [from 3.39] + / [from 3.16] h: object [clinic start generated code]*/ @@ -2265,7 +2265,7 @@ static PyObject * depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g, PyObject *h) -/*[clinic end generated code: output=ddfbde80fe1942e1 input=a84c447a74284174]*/ +/*[clinic end generated code: output=ddfbde80fe1942e1 input=7a074e621c79efd7]*/ { Py_RETURN_NONE; } @@ -2276,13 +2276,13 @@ depr_multi a: object / b: object - / [from 3.37] + / [from 3.14] c: object - / [from 3.38] + / [from 3.15] d: object - * [from 3.38] + * [from 3.15] e: object - * [from 3.37] + * [from 3.14] f: object * g: object @@ -2291,7 +2291,7 @@ depr_multi static PyObject * depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g) -/*[clinic end generated code: output=f81c92852ca2d4ee input=0e859e8b8eda75d7]*/ +/*[clinic end generated code: output=f81c92852ca2d4ee input=5b847c5e44bedd02]*/ { Py_RETURN_NONE; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index fdf22a0c994d3a..243c7346576fc6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -121,7 +121,7 @@ get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args)) PyThreadState *tstate = _PyThreadState_GET(); uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - int remaining = (int)((here_addr - _tstate->c_stack_soft_limit)/PYOS_STACK_MARGIN_BYTES * 50); + int remaining = (int)((here_addr - _tstate->c_stack_soft_limit) / _PyOS_STACK_MARGIN_BYTES * 50); return PyLong_FromLong(remaining); } @@ -2207,8 +2207,7 @@ get_code(PyObject *obj) return (PyCodeObject *)PyFunction_GetCode(obj); } return (PyCodeObject *)PyErr_Format( - PyExc_TypeError, "expected function or code object, got %s", - Py_TYPE(obj)->tp_name); + PyExc_TypeError, "expected function or code object, got %T", obj); } static PyObject * @@ -2346,6 +2345,37 @@ incref_decref_delayed(PyObject *self, PyObject *op) Py_RETURN_NONE; } +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" + +EM_JS(int, emscripten_set_up_async_input_device_js, (void), { + let idx = 0; + const encoder = new TextEncoder(); + const bufs = [ + encoder.encode("ab\n"), + encoder.encode("fi\n"), + encoder.encode("xy\n"), + ]; + function sleep(t) { + return new Promise(res => setTimeout(res, t)); + } + FS.createAsyncInputDevice("/dev", "blah", async () => { + await sleep(5); + return bufs[(idx ++) % 3]; + }); + return !!WebAssembly.promising; +}); + +static PyObject * +emscripten_set_up_async_input_device(PyObject *self, PyObject *Py_UNUSED(ignored)) { + if (emscripten_set_up_async_input_device_js()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +#endif + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2448,6 +2478,9 @@ static PyMethodDef module_functions[] = { {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF +#ifdef __EMSCRIPTEN__ + {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 150a266b521736..f82ad6870f850f 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -681,12 +681,12 @@ PyThreadHandleObject_join(PyObject *op, PyObject *args) PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); PyObject *timeout_obj = NULL; - if (!PyArg_ParseTuple(args, "|O?:join", &timeout_obj)) { + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } PyTime_t timeout_ns = -1; - if (timeout_obj != NULL) { + if (timeout_obj != NULL && timeout_obj != Py_None) { if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -834,9 +834,14 @@ lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds) return NULL; } - PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout, - _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + PyLockStatus r = _PyMutex_LockTimed( + &self->lock, timeout, + _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { + assert(PyErr_Occurred()); + return NULL; + } + if (r == PY_LOCK_FAILURE && PyErr_Occurred()) { return NULL; } @@ -1054,9 +1059,14 @@ rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds) return NULL; } - PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout, - _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); + PyLockStatus r = _PyRecursiveMutex_LockTimed( + &self->lock, timeout, + _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH); if (r == PY_LOCK_INTR) { + assert(PyErr_Occurred()); + return NULL; + } + if (r == PY_LOCK_FAILURE && PyErr_Occurred()) { return NULL; } @@ -1365,9 +1375,7 @@ static void localdummy_dealloc(PyObject *op) { localdummyobject *self = localdummyobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1949,10 +1957,10 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, PyObject *func = NULL; int daemon = 1; thread_module_state *state = get_thread_state(module); - PyObject *hobj = Py_None; + PyObject *hobj = NULL; if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, - "O|O!?p:start_joinable_thread", keywords, - &func, state->thread_handle_type, &hobj, &daemon)) { + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { return NULL; } @@ -1962,6 +1970,14 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, return NULL; } + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, hobj) < 0) { return NULL; @@ -2322,7 +2338,7 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +"_excepthook($module, args, /)\n\ --\n\ \n\ Handle uncaught Thread.run() exception."); diff --git a/Modules/_typesmodule.c b/Modules/_typesmodule.c index a30a88196e7192..df6b4c93cb87a6 100644 --- a/Modules/_typesmodule.c +++ b/Modules/_typesmodule.c @@ -28,6 +28,7 @@ _types_exec(PyObject *m) EXPORT_STATIC_TYPE("CoroutineType", PyCoro_Type); EXPORT_STATIC_TYPE("EllipsisType", PyEllipsis_Type); EXPORT_STATIC_TYPE("FrameType", PyFrame_Type); + EXPORT_STATIC_TYPE("FrameLocalsProxyType", PyFrameLocalsProxy_Type); EXPORT_STATIC_TYPE("FunctionType", PyFunction_Type); EXPORT_STATIC_TYPE("GeneratorType", PyGen_Type); EXPORT_STATIC_TYPE("GenericAlias", Py_GenericAliasType); diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index abd53436b21b29..5c5383d260a040 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -7,6 +7,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "datetime.h" // PyDateTime_TZInfo @@ -375,9 +376,7 @@ zoneinfo_dealloc(PyObject *obj_self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(obj_self); - } + FT_CLEAR_WEAKREFS(obj_self, self->weakreflist); if (self->trans_list_utc != NULL) { PyMem_Free(self->trans_list_utc); diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index d75c0779474a82..8af6156a0da575 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 4e8f708f2232c7..82226ff8718e6b 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 4c885fa0d720fd..0ac7bcb4ddc416 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index bc9e6eff89af68..508b136817872b 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdCompressor class definitions */ diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index c53d6e4cb05cf0..b00ee05d2f51bf 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDecompressor class definition */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 14f74aaed46ec5..35d6ca8e55a265 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDict class definitions */ diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index 4a403416dbd4a3..e0d3f46b2b14a6 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -1,4 +1,4 @@ -/* Low level interface to the Zstandard algorthm & the zstd library. */ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef ZSTD_DICT_H #define ZSTD_DICT_H diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 401a3a7072b846..5d07de2fba9526 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,6 +13,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include <stdbool.h> @@ -728,9 +729,7 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->ob_item != NULL) { PyMem_Free(self->ob_item); } diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 49c864318c8aa8..a8c32d6510604a 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -2372,7 +2372,22 @@ _curses_ungetmouse(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("ungetmouse", "argument 5", "int", args[4]); goto exit; } - bstate = PyLong_AsUnsignedLongMask(args[4]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[4], &bstate, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = _curses_ungetmouse_impl(module, id, x, y, z, bstate); exit: @@ -3138,7 +3153,22 @@ _curses_mousemask(PyObject *module, PyObject *arg) _PyArg_BadArgument("mousemask", "argument", "int", arg); goto exit; } - newmask = PyLong_AsUnsignedLongMask(arg); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(arg, &newmask, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = _curses_mousemask_impl(module, newmask); exit: @@ -4420,4 +4450,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=a083473003179b30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=79ddaae4da3b80df input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 61ea10e2a48284..3d81a6dcce1440 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1492,8 +1492,6 @@ pbkdf2_hmac(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } -#if defined(PY_OPENSSL_HAS_SCRYPT) - PyDoc_STRVAR(_hashlib_scrypt__doc__, "scrypt($module, /, password, *, salt, n, r, p, maxmem=0, dklen=64)\n" "--\n" @@ -1601,8 +1599,6 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -#endif /* defined(PY_OPENSSL_HAS_SCRYPT) */ - PyDoc_STRVAR(_hashlib_hmac_singleshot__doc__, "hmac_digest($module, /, key, msg, digest)\n" "--\n" @@ -1980,8 +1976,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #define _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #endif /* !defined(_HASHLIB_OPENSSL_SHAKE_256_METHODDEF) */ - -#ifndef _HASHLIB_SCRYPT_METHODDEF - #define _HASHLIB_SCRYPT_METHODDEF -#endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=29f4aaf01714778e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cd5ff436f6dc2938 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index e80b24b54c0ffa..f6a51cdba6b401 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -235,26 +235,41 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "get_async_stack_trace($self, /)\n" "--\n" "\n" -"Returns information about the currently running async task and its stack trace.\n" +"Get the currently running async tasks and their dependency graphs from the remote process.\n" "\n" -"Returns a tuple of (task_info, stack_frames) where:\n" -"- task_info is a tuple of (task_id, task_name) identifying the task\n" -"- stack_frames is a list of tuples (function_name, filename, line_number) representing\n" -" the Python stack frames for the task, ordered from most recent to oldest\n" +"This returns information about running tasks and all tasks that are waiting for them,\n" +"forming a complete dependency graph for each thread\'s active task.\n" "\n" -"Example:\n" -" ((4345585712, \'Task-1\'), [\n" -" (\'run_echo_server\', \'server.py\', 127),\n" -" (\'serve_forever\', \'server.py\', 45),\n" -" (\'main\', \'app.py\', 23)\n" -" ])\n" +"For each thread with a running task, returns the running task plus all tasks that\n" +"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" +"those tasks, etc.).\n" +"\n" +"Returns a list of per-thread results, where each thread result contains:\n" +"- Thread ID\n" +"- List of task information for the running task and all its waiters\n" +"\n" +"Each task info contains:\n" +"- Task ID (memory address)\n" +"- Task name\n" +"- Call stack frames: List of (func_name, filename, lineno)\n" +"- List of tasks waiting for this task (recursive structure)\n" "\n" "Raises:\n" " RuntimeError: If AsyncioDebug section is not available in the target process\n" -" RuntimeError: If there is an error copying memory from the target process\n" -" OSError: If there is an error accessing the target process\n" -" PermissionError: If access to the target process is denied\n" -" UnicodeDecodeError: If there is an error decoding strings from the target process"); +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"[\n" +" (140234, [\n" +" (4345585712, \'main_task\',\n" +" [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n" +" [\n" +" (4345585800, \'worker_1\', [...], [...]),\n" +" (4345585900, \'worker_2\', [...], [...])\n" +" ])\n" +" ])\n" +"]"); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, @@ -273,4 +288,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=a37ab223d5081b16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0dd1e6e8bab2a8b1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index c6e2abd4d93474..7027d87379283d 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -442,6 +443,115 @@ _ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(conte return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_send__doc__, +"uses_ktls_for_send($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for sending."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF \ + {"uses_ktls_for_send", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_send, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_send__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_send_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_recv__doc__, +"uses_ktls_for_recv($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for receiving."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF \ + {"uses_ktls_for_recv", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_recv, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_recv__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_recv_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(BIO_get_ktls_send) + +PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, +"sendfile($self, fd, offset, size, flags=0, /)\n" +"--\n" +"\n" +"Write size bytes from offset in the file descriptor fd to the SSL connection.\n" +"\n" +"This method uses the zero-copy technique and returns the number of bytes\n" +"written. It should be called only when Kernel TLS is used for sending data in\n" +"the connection.\n" +"\n" +"The meaning of flags is platform dependent."); + +#define _SSL__SSLSOCKET_SENDFILE_METHODDEF \ + {"sendfile", _PyCFunction_CAST(_ssl__SSLSocket_sendfile), METH_FASTCALL, _ssl__SSLSocket_sendfile__doc__}, + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags); + +static PyObject * +_ssl__SSLSocket_sendfile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + Py_off_t offset; + size_t size; + int flags = 0; + + if (!_PyArg_CheckPositional("sendfile", nargs, 3, 4)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + if (!Py_off_t_converter(args[1], &offset)) { + goto exit; + } + if (!_PyLong_Size_t_Converter(args[2], &size)) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + flags = PyLong_AsInt(args[3]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_sendfile_impl((PySSLSocket *)self, fd, offset, size, flags); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(BIO_get_ktls_send) */ + PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, "write($self, b, /)\n" "--\n" @@ -2893,6 +3003,10 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #endif /* defined(_MSC_VER) */ +#ifndef _SSL__SSLSOCKET_SENDFILE_METHODDEF + #define _SSL__SSLSOCKET_SENDFILE_METHODDEF +#endif /* !defined(_SSL__SSLSOCKET_SENDFILE_METHODDEF) */ + #ifndef _SSL_ENUM_CERTIFICATES_METHODDEF #define _SSL_ENUM_CERTIFICATES_METHODDEF #endif /* !defined(_SSL_ENUM_CERTIFICATES_METHODDEF) */ @@ -2900,4 +3014,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1adc3780d8ca682a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 970528ce9ea46d..68c92a86226bc9 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -746,12 +746,19 @@ unsigned_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional; } { - unsigned long ival = PyLong_AsUnsignedLongMask(args[2]); - if (ival == (unsigned long)-1 && PyErr_Occurred()) { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { goto exit; } - else { - c = (unsigned char) ival; + if ((size_t)_bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } } } skip_optional: @@ -848,9 +855,21 @@ unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - c = (unsigned short)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned short)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned short), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned short)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = unsigned_short_converter_impl(module, a, b, c); @@ -955,9 +974,21 @@ unsigned_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs if (nargs < 3) { goto skip_optional; } - c = (unsigned int)PyLong_AsUnsignedLongMask(args[2]); - if (c == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = unsigned_int_converter_impl(module, a, b, c); @@ -1042,7 +1073,22 @@ unsigned_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t narg _PyArg_BadArgument("unsigned_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = unsigned_long_converter_impl(module, a, b, c); @@ -1126,7 +1172,22 @@ unsigned_long_long_converter(PyObject *module, PyObject *const *args, Py_ssize_t _PyArg_BadArgument("unsigned_long_long_converter", "argument 3", "int", args[2]); goto exit; } - c = PyLong_AsUnsignedLongLongMask(args[2]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[2], &c, sizeof(unsigned long long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } skip_optional: return_value = unsigned_long_long_converter_impl(module, a, b, c); @@ -4481,4 +4542,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=84ffc31f27215baa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b04671afdafbecf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 135197f06fd7b4..a46d238801b321 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -19,16 +19,16 @@ PyDoc_STRVAR(depr_star_new__doc__, "\n" "Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" "deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); static PyObject * depr_star_new_impl(PyTypeObject *type, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprStarNew.__new__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprStarNew.__new__'.") # else @@ -77,7 +77,7 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to _testclinic.DeprStarNew() is " "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -104,7 +104,7 @@ PyDoc_STRVAR(depr_star_new_clone__doc__, "\n" "Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" "is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_STAR_NEW_CLONE_METHODDEF \ @@ -113,10 +113,10 @@ PyDoc_STRVAR(depr_star_new_clone__doc__, static PyObject * depr_star_new_clone_impl(PyObject *type, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprStarNew.cloned'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprStarNew.cloned'.") # else @@ -163,7 +163,7 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to _testclinic.DeprStarNew.cloned()" " is deprecated. Parameter 'a' will become a keyword-only " - "parameter in Python 3.37.", 1)) + "parameter in Python 3.14.", 1)) { goto exit; } @@ -192,16 +192,16 @@ PyDoc_STRVAR(depr_star_init__doc__, "\n" "Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" "deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); static int depr_star_init_impl(PyObject *self, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprStarInit.__init__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprStarInit.__init__'.") # else @@ -250,7 +250,7 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to _testclinic.DeprStarInit() is " "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -277,7 +277,7 @@ PyDoc_STRVAR(depr_star_init_clone__doc__, "\n" "Note: Passing positional arguments to\n" "_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" -"become a keyword-only parameter in Python 3.37.\n" +"become a keyword-only parameter in Python 3.14.\n" ""); #define DEPR_STAR_INIT_CLONE_METHODDEF \ @@ -286,10 +286,10 @@ PyDoc_STRVAR(depr_star_init_clone__doc__, static PyObject * depr_star_init_clone_impl(PyObject *self, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprStarInit.cloned'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprStarInit.cloned'.") # else @@ -336,7 +336,7 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to " "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " - "will become a keyword-only parameter in Python 3.37.", 1)) + "will become a keyword-only parameter in Python 3.14.", 1)) { goto exit; } @@ -361,10 +361,10 @@ static int depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'.") # else @@ -414,7 +414,7 @@ depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to " "_testclinic.DeprStarInitNoInline() is deprecated. Parameters 'b'" - " and 'c' will become keyword-only parameters in Python 3.37.", 1)) + " and 'c' will become keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -436,16 +436,16 @@ PyDoc_STRVAR(depr_kwd_new__doc__, "The deprecation message should use the class name instead of __new__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdNew() is\n" -"deprecated. Parameter \'a\' will become positional-only in Python 3.37.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static PyObject * depr_kwd_new_impl(PyTypeObject *type, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprKwdNew.__new__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprKwdNew.__new__'.") # else @@ -499,7 +499,7 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdNew() is " "deprecated. Parameter 'a' will become positional-only in Python " - "3.37.", 1)) + "3.14.", 1)) { goto exit; } @@ -522,16 +522,16 @@ PyDoc_STRVAR(depr_kwd_init__doc__, "The deprecation message should use the class name instead of __init__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdInit() is\n" -"deprecated. Parameter \'a\' will become positional-only in Python 3.37.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static int depr_kwd_init_impl(PyObject *self, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprKwdInit.__init__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprKwdInit.__init__'.") # else @@ -585,7 +585,7 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdInit() is " "deprecated. Parameter 'a' will become positional-only in Python " - "3.37.", 1)) + "3.14.", 1)) { goto exit; } @@ -605,10 +605,10 @@ static int depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'.") # else @@ -665,7 +665,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " "_testclinic.DeprKwdInitNoInline() is deprecated. Parameters 'b' " - "and 'c' will become positional-only in Python 3.37.", 1)) + "and 'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -682,7 +682,7 @@ PyDoc_STRVAR(depr_star_pos0_len1__doc__, "\n" "Note: Passing positional arguments to depr_star_pos0_len1() is\n" "deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_STAR_POS0_LEN1_METHODDEF \ @@ -691,10 +691,10 @@ PyDoc_STRVAR(depr_star_pos0_len1__doc__, static PyObject * depr_star_pos0_len1_impl(PyObject *module, PyObject *a); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos0_len1'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos0_len1'.") # else @@ -740,7 +740,7 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to depr_star_pos0_len1() is " "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -763,7 +763,7 @@ PyDoc_STRVAR(depr_star_pos0_len2__doc__, "\n" "Note: Passing positional arguments to depr_star_pos0_len2() is\n" "deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" -"in Python 3.37.\n" +"in Python 3.14.\n" ""); #define DEPR_STAR_POS0_LEN2_METHODDEF \ @@ -772,10 +772,10 @@ PyDoc_STRVAR(depr_star_pos0_len2__doc__, static PyObject * depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos0_len2'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos0_len2'.") # else @@ -822,7 +822,7 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to depr_star_pos0_len2() is " "deprecated. Parameters 'a' and 'b' will become keyword-only " - "parameters in Python 3.37.", 1)) + "parameters in Python 3.14.", 1)) { goto exit; } @@ -846,7 +846,7 @@ PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, "\n" "Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" "is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" -"parameters in Python 3.37.\n" +"parameters in Python 3.14.\n" ""); #define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ @@ -856,10 +856,10 @@ static PyObject * depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos0_len3_with_kwd'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos0_len3_with_kwd'.") # else @@ -908,7 +908,7 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to depr_star_pos0_len3_with_kwd() " "is deprecated. Parameters 'a', 'b' and 'c' will become " - "keyword-only parameters in Python 3.37.", 1)) + "keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -934,7 +934,7 @@ PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, "\n" "Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" "deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ @@ -943,10 +943,10 @@ PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, static PyObject * depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos1_len1_opt'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos1_len1_opt'.") # else @@ -994,7 +994,7 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -1022,7 +1022,7 @@ PyDoc_STRVAR(depr_star_pos1_len1__doc__, "\n" "Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" "deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_STAR_POS1_LEN1_METHODDEF \ @@ -1031,10 +1031,10 @@ PyDoc_STRVAR(depr_star_pos1_len1__doc__, static PyObject * depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos1_len1'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos1_len1'.") # else @@ -1081,7 +1081,7 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to depr_star_pos1_len1() is " "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -1105,7 +1105,7 @@ PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, "\n" "Note: Passing more than 1 positional argument to\n" "depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" -"will become keyword-only parameters in Python 3.37.\n" +"will become keyword-only parameters in Python 3.14.\n" ""); #define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ @@ -1115,10 +1115,10 @@ static PyObject * depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos1_len2_with_kwd'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos1_len2_with_kwd'.") # else @@ -1167,7 +1167,7 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to " "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" - " 'c' will become keyword-only parameters in Python 3.37.", 1)) + " 'c' will become keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -1193,7 +1193,7 @@ PyDoc_STRVAR(depr_star_pos2_len1__doc__, "\n" "Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" "deprecated. Parameter \'c\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_STAR_POS2_LEN1_METHODDEF \ @@ -1203,10 +1203,10 @@ static PyObject * depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos2_len1'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos2_len1'.") # else @@ -1254,7 +1254,7 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 3 positional arguments to depr_star_pos2_len1() is " "deprecated. Parameter 'c' will become a keyword-only parameter " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -1279,7 +1279,7 @@ PyDoc_STRVAR(depr_star_pos2_len2__doc__, "\n" "Note: Passing more than 2 positional arguments to\n" "depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" -"become keyword-only parameters in Python 3.37.\n" +"become keyword-only parameters in Python 3.14.\n" ""); #define DEPR_STAR_POS2_LEN2_METHODDEF \ @@ -1289,10 +1289,10 @@ static PyObject * depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos2_len2'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos2_len2'.") # else @@ -1341,7 +1341,7 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to " "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" - " become keyword-only parameters in Python 3.37.", 1)) + " become keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -1367,7 +1367,7 @@ PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, "\n" "Note: Passing more than 2 positional arguments to\n" "depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" -"will become keyword-only parameters in Python 3.37.\n" +"will become keyword-only parameters in Python 3.14.\n" ""); #define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ @@ -1377,10 +1377,10 @@ static PyObject * depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_pos2_len2_with_kwd'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_pos2_len2_with_kwd'.") # else @@ -1430,7 +1430,7 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to " "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" - " 'd' will become keyword-only parameters in Python 3.37.", 1)) + " 'd' will become keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -1457,7 +1457,7 @@ PyDoc_STRVAR(depr_star_noinline__doc__, "\n" "Note: Passing more than 1 positional argument to depr_star_noinline()\n" "is deprecated. Parameters \'b\' and \'c\' will become keyword-only\n" -"parameters in Python 3.37.\n" +"parameters in Python 3.14.\n" ""); #define DEPR_STAR_NOINLINE_METHODDEF \ @@ -1467,10 +1467,10 @@ static PyObject * depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_noinline'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_noinline'.") # else @@ -1519,7 +1519,7 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to depr_star_noinline() " "is deprecated. Parameters 'b' and 'c' will become keyword-only " - "parameters in Python 3.37.", 1)) + "parameters in Python 3.14.", 1)) { goto exit; } @@ -1540,9 +1540,9 @@ PyDoc_STRVAR(depr_star_multi__doc__, "\n" "Note: Passing more than 1 positional argument to depr_star_multi() is\n" "deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.39. Parameters \'c\' and \'d\' will become keyword-only\n" -"parameters in Python 3.38. Parameters \'e\', \'f\' and \'g\' will become\n" -"keyword-only parameters in Python 3.37.\n" +"Python 3.16. Parameters \'c\' and \'d\' will become keyword-only\n" +"parameters in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"keyword-only parameters in Python 3.14.\n" ""); #define DEPR_STAR_MULTI_METHODDEF \ @@ -1553,10 +1553,10 @@ depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g, PyObject *h); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_star_multi'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_star_multi'.") # else @@ -1609,9 +1609,9 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to depr_star_multi() is " "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.39. Parameters 'c' and 'd' will become keyword-only " - "parameters in Python 3.38. Parameters 'e', 'f' and 'g' will " - "become keyword-only parameters in Python 3.37.", 1)) + "in Python 3.16. Parameters 'c' and 'd' will become keyword-only " + "parameters in Python 3.15. Parameters 'e', 'f' and 'g' will " + "become keyword-only parameters in Python 3.14.", 1)) { goto exit; } @@ -1640,7 +1640,7 @@ PyDoc_STRVAR(depr_kwd_required_1__doc__, "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_required_1() is\n" -"deprecated. Parameter \'b\' will become positional-only in Python 3.37.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_REQUIRED_1_METHODDEF \ @@ -1649,10 +1649,10 @@ PyDoc_STRVAR(depr_kwd_required_1__doc__, static PyObject * depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_required_1'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_required_1'.") # else @@ -1704,7 +1704,7 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_required_1() is " "deprecated. Parameter 'b' will become positional-only in Python " - "3.37.", 1)) + "3.14.", 1)) { goto exit; } @@ -1723,7 +1723,7 @@ PyDoc_STRVAR(depr_kwd_required_2__doc__, "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_required_2()\n" "is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_KWD_REQUIRED_2_METHODDEF \ @@ -1733,10 +1733,10 @@ static PyObject * depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_required_2'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_required_2'.") # else @@ -1789,7 +1789,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_required_2() " "is deprecated. Parameters 'b' and 'c' will become " - "positional-only in Python 3.37.", 1)) + "positional-only in Python 3.14.", 1)) { goto exit; } @@ -1808,7 +1808,7 @@ PyDoc_STRVAR(depr_kwd_optional_1__doc__, "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_optional_1() is\n" -"deprecated. Parameter \'b\' will become positional-only in Python 3.37.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_OPTIONAL_1_METHODDEF \ @@ -1817,10 +1817,10 @@ PyDoc_STRVAR(depr_kwd_optional_1__doc__, static PyObject * depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_optional_1'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_optional_1'.") # else @@ -1873,7 +1873,7 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_optional_1() is " "deprecated. Parameter 'b' will become positional-only in Python " - "3.37.", 1)) + "3.14.", 1)) { goto exit; } @@ -1896,7 +1896,7 @@ PyDoc_STRVAR(depr_kwd_optional_2__doc__, "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_optional_2()\n" "is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_KWD_OPTIONAL_2_METHODDEF \ @@ -1906,10 +1906,10 @@ static PyObject * depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_optional_2'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_optional_2'.") # else @@ -1963,7 +1963,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_optional_2() " "is deprecated. Parameters 'b' and 'c' will become " - "positional-only in Python 3.37.", 1)) + "positional-only in Python 3.14.", 1)) { goto exit; } @@ -1992,7 +1992,7 @@ PyDoc_STRVAR(depr_kwd_optional_3__doc__, "\n" "Note: Passing keyword arguments \'a\', \'b\' and \'c\' to\n" "depr_kwd_optional_3() is deprecated. Parameters \'a\', \'b\' and \'c\' will\n" -"become positional-only in Python 3.37.\n" +"become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_OPTIONAL_3_METHODDEF \ @@ -2002,10 +2002,10 @@ static PyObject * depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_optional_3'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_optional_3'.") # else @@ -2059,7 +2059,7 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'a', 'b' and 'c' to " "depr_kwd_optional_3() is deprecated. Parameters 'a', 'b' and 'c'" - " will become positional-only in Python 3.37.", 1)) + " will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -2093,7 +2093,7 @@ PyDoc_STRVAR(depr_kwd_required_optional__doc__, "\n" "Note: Passing keyword arguments \'b\' and \'c\' to\n" "depr_kwd_required_optional() is deprecated. Parameters \'b\' and \'c\'\n" -"will become positional-only in Python 3.37.\n" +"will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF \ @@ -2103,10 +2103,10 @@ static PyObject * depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_required_optional'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_required_optional'.") # else @@ -2160,7 +2160,7 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " "depr_kwd_required_optional() is deprecated. Parameters 'b' and " - "'c' will become positional-only in Python 3.37.", 1)) + "'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -2184,7 +2184,7 @@ PyDoc_STRVAR(depr_kwd_noinline__doc__, "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_noinline() is\n" "deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" -"Python 3.37.\n" +"Python 3.14.\n" ""); #define DEPR_KWD_NOINLINE_METHODDEF \ @@ -2194,10 +2194,10 @@ static PyObject * depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, const char *d, Py_ssize_t d_length); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_noinline'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_noinline'.") # else @@ -2253,7 +2253,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_noinline() is " "deprecated. Parameters 'b' and 'c' will become positional-only " - "in Python 3.37.", 1)) + "in Python 3.14.", 1)) { goto exit; } @@ -2270,9 +2270,9 @@ PyDoc_STRVAR(depr_kwd_multi__doc__, "\n" "Note: Passing keyword arguments \'b\', \'c\', \'d\', \'e\', \'f\' and \'g\' to\n" "depr_kwd_multi() is deprecated. Parameter \'b\' will become positional-\n" -"only in Python 3.37. Parameters \'c\' and \'d\' will become positional-\n" -"only in Python 3.38. Parameters \'e\', \'f\' and \'g\' will become\n" -"positional-only in Python 3.39.\n" +"only in Python 3.14. Parameters \'c\' and \'d\' will become positional-\n" +"only in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"positional-only in Python 3.16.\n" ""); #define DEPR_KWD_MULTI_METHODDEF \ @@ -2283,10 +2283,10 @@ depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g, PyObject *h); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_kwd_multi'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_kwd_multi'.") # else @@ -2344,9 +2344,9 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to " "depr_kwd_multi() is deprecated. Parameter 'b' will become " - "positional-only in Python 3.37. Parameters 'c' and 'd' will " - "become positional-only in Python 3.38. Parameters 'e', 'f' and " - "'g' will become positional-only in Python 3.39.", 1)) + "positional-only in Python 3.14. Parameters 'c' and 'd' will " + "become positional-only in Python 3.15. Parameters 'e', 'f' and " + "'g' will become positional-only in Python 3.16.", 1)) { goto exit; } @@ -2370,14 +2370,14 @@ PyDoc_STRVAR(depr_multi__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_multi() is\n" -"deprecated. Parameter \'b\' will become positional-only in Python 3.37.\n" -"Parameter \'c\' will become positional-only in Python 3.38.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" +"Parameter \'c\' will become positional-only in Python 3.15.\n" "\n" "\n" "Note: Passing more than 4 positional arguments to depr_multi() is\n" "deprecated. Parameter \'e\' will become a keyword-only parameter in\n" -"Python 3.38. Parameter \'f\' will become a keyword-only parameter in\n" -"Python 3.37.\n" +"Python 3.15. Parameter \'f\' will become a keyword-only parameter in\n" +"Python 3.14.\n" ""); #define DEPR_MULTI_METHODDEF \ @@ -2387,10 +2387,10 @@ static PyObject * depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d, PyObject *e, PyObject *f, PyObject *g); -// Emit compiler warnings when we get to Python 3.37. -#if PY_VERSION_HEX >= 0x032500C0 +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 # error "Update the clinic input of 'depr_multi'." -#elif PY_VERSION_HEX >= 0x032500A0 +#elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER # pragma message ("Update the clinic input of 'depr_multi'.") # else @@ -2442,8 +2442,8 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 4 positional arguments to depr_multi() is " "deprecated. Parameter 'e' will become a keyword-only parameter " - "in Python 3.38. Parameter 'f' will become a keyword-only " - "parameter in Python 3.37.", 1)) + "in Python 3.15. Parameter 'f' will become a keyword-only " + "parameter in Python 3.14.", 1)) { goto exit; } @@ -2457,7 +2457,7 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_multi() is " "deprecated. Parameter 'b' will become positional-only in Python " - "3.37. Parameter 'c' will become positional-only in Python 3.38.", 1)) + "3.14. Parameter 'c' will become positional-only in Python 3.15.", 1)) { goto exit; } @@ -2474,4 +2474,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * exit: return return_value; } -/*[clinic end generated code: output=517bb49913bafc4a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4e60af44fd6b7b94 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 09ac157cbfd135..19564a6c13f246 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -434,12 +434,19 @@ zoneinfo_ZoneInfo__unpickle(PyObject *type, PyTypeObject *cls, PyObject *const * } key = args[0]; { - unsigned long ival = PyLong_AsUnsignedLongMask(args[1]); - if (ival == (unsigned long)-1 && PyErr_Occurred()) { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &from_cache, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { goto exit; } - else { - from_cache = (unsigned char) ival; + if ((size_t)_bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } } } return_value = zoneinfo_ZoneInfo__unpickle_impl((PyTypeObject *)type, cls, key, from_cache); @@ -447,4 +454,4 @@ zoneinfo_ZoneInfo__unpickle(PyObject *type, PyTypeObject *cls, PyObject *const * exit: return return_value; } -/*[clinic end generated code: output=8e9e204f390261b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6df04d7b400bd7f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 602e42a4c1aaa4..ce29e0d11a45cd 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -292,9 +292,21 @@ binascii_crc_hqx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (PyObject_GetBuffer(args[0], &data, PyBUF_SIMPLE) != 0) { goto exit; } - crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (crc == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = binascii_crc_hqx_impl(module, &data, crc); @@ -336,9 +348,21 @@ binascii_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - crc = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (crc == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: _return_value = binascii_crc32_impl(module, &data, crc); @@ -788,4 +812,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=adb855a2797c3cad input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fba6a71e0d7d092f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 00a929064ba700..005e9b9e12afd9 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -124,7 +124,22 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); goto exit; } - code = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &code, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } if (nargs < 3) { goto skip_optional; } @@ -264,4 +279,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=65a16bc64c7b4de4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf84289b741e7cf6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index a443c48faaa88a..246019f2206028 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -84,6 +84,112 @@ PyDoc_STRVAR(math_floor__doc__, #define MATH_FLOOR_METHODDEF \ {"floor", (PyCFunction)math_floor, METH_O, math_floor__doc__}, +PyDoc_STRVAR(math_fmax__doc__, +"fmax($module, x, y, /)\n" +"--\n" +"\n" +"Return the larger of two floating-point arguments."); + +#define MATH_FMAX_METHODDEF \ + {"fmax", _PyCFunction_CAST(math_fmax), METH_FASTCALL, math_fmax__doc__}, + +static double +math_fmax_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmax(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmax", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmax_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(math_fmin__doc__, +"fmin($module, x, y, /)\n" +"--\n" +"\n" +"Return the smaller of two floating-point arguments."); + +#define MATH_FMIN_METHODDEF \ + {"fmin", _PyCFunction_CAST(math_fmin), METH_FASTCALL, math_fmin__doc__}, + +static double +math_fmin_impl(PyObject *module, double x, double y); + +static PyObject * +math_fmin(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + double x; + double y; + double _return_value; + + if (!_PyArg_CheckPositional("fmin", nargs, 2, 2)) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + x = PyFloat_AS_DOUBLE(args[0]); + } + else + { + x = PyFloat_AsDouble(args[0]); + if (x == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + if (PyFloat_CheckExact(args[1])) { + y = PyFloat_AS_DOUBLE(args[1]); + } + else + { + y = PyFloat_AsDouble(args[1]); + if (y == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + _return_value = math_fmin_impl(module, x, y); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(math_signbit__doc__, "signbit($module, x, /)\n" "--\n" @@ -1212,4 +1318,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4e3fa94d026f027b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4fb180d4c25ff8fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 3621a0625411d3..8af9e1db781c8f 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -380,7 +380,7 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "path may always be specified as a string.\n" "On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"If this functionality is unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -844,7 +844,22 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * _PyArg_BadArgument("chflags", "argument 'flags'", "int", args[1]); goto exit; } - flags = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } if (!noptargs) { goto skip_optional_pos; } @@ -928,7 +943,22 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject _PyArg_BadArgument("lchflags", "argument 'flags'", "int", args[1]); goto exit; } - flags = PyLong_AsUnsignedLongMask(args[1]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } return_value = os_lchflags_impl(module, &path, flags); exit: @@ -2014,57 +2044,24 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_splitroot__doc__, -"_path_splitroot($module, /, path)\n" +"_path_splitroot($module, path, /)\n" "--\n" "\n" "Removes everything after the root on Win32."); #define OS__PATH_SPLITROOT_METHODDEF \ - {"_path_splitroot", _PyCFunction_CAST(os__path_splitroot), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot__doc__}, + {"_path_splitroot", (PyCFunction)os__path_splitroot, METH_O, os__path_splitroot__doc__}, static PyObject * os__path_splitroot_impl(PyObject *module, path_t *path); static PyObject * -os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_splitroot(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_splitroot", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } return_value = os__path_splitroot_impl(module, &path); @@ -2225,58 +2222,25 @@ os__path_lexists(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #if defined(MS_WINDOWS) PyDoc_STRVAR(os__path_isdir__doc__, -"_path_isdir($module, /, s)\n" +"_path_isdir($module, path, /)\n" "--\n" "\n" "Return true if the pathname refers to an existing directory."); #define OS__PATH_ISDIR_METHODDEF \ - {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, + {"_path_isdir", (PyCFunction)os__path_isdir, METH_O, os__path_isdir__doc__}, static int os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * -os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_isdir(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { _Py_LATIN1_CHR('s'), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_isdir", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } _return_value = os__path_isdir_impl(module, &path); @@ -2511,7 +2475,7 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P #endif /* defined(MS_WINDOWS) */ PyDoc_STRVAR(os__path_splitroot_ex__doc__, -"_path_splitroot_ex($module, /, p)\n" +"_path_splitroot_ex($module, path, /)\n" "--\n" "\n" "Split a pathname into drive, root and tail.\n" @@ -2519,51 +2483,18 @@ PyDoc_STRVAR(os__path_splitroot_ex__doc__, "The tail contains anything after the root."); #define OS__PATH_SPLITROOT_EX_METHODDEF \ - {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, + {"_path_splitroot_ex", (PyCFunction)os__path_splitroot_ex, METH_O, os__path_splitroot_ex__doc__}, static PyObject * os__path_splitroot_ex_impl(PyObject *module, path_t *path); static PyObject * -os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_splitroot_ex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { _Py_LATIN1_CHR('p'), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"p", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_splitroot_ex", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!path_converter(args[0], &path)) { + if (!path_converter(arg, &path)) { goto exit; } return_value = os__path_splitroot_ex_impl(module, &path); @@ -11373,9 +11304,21 @@ os_memfd_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_pos; } - flags = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (flags == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &flags, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional_pos: return_value = os_memfd_create_impl(module, name, flags); @@ -12727,6 +12670,80 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(__EMSCRIPTEN__) */ +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_log__doc__, +"_emscripten_log($module, /, arg)\n" +"--\n" +"\n" +"Log something to the JS console. Emscripten only."); + +#define OS__EMSCRIPTEN_LOG_METHODDEF \ + {"_emscripten_log", _PyCFunction_CAST(os__emscripten_log), METH_FASTCALL|METH_KEYWORDS, os__emscripten_log__doc__}, + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg); + +static PyObject * +os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_emscripten_log", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_emscripten_log", "argument 'arg'", "str", args[0]); + goto exit; + } + Py_ssize_t arg_length; + arg = PyUnicode_AsUTF8AndSize(args[0], &arg_length); + if (arg == NULL) { + goto exit; + } + if (strlen(arg) != (size_t)arg_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = os__emscripten_log_impl(module, arg); + +exit: + return return_value; +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13398,4 +13415,8 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=ae64df0389746258 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_LOG_METHODDEF + #define OS__EMSCRIPTEN_LOG_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ +/*[clinic end generated code: output=b1e2615384347102 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pwdmodule.c.h b/Modules/clinic/pwdmodule.c.h index 365d99aab1dd22..43d4825031c7e6 100644 --- a/Modules/clinic/pwdmodule.c.h +++ b/Modules/clinic/pwdmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_BadArgument() + PyDoc_STRVAR(pwd_getpwuid__doc__, "getpwuid($module, uidobj, /)\n" "--\n" @@ -34,7 +36,7 @@ pwd_getpwnam(PyObject *module, PyObject *arg) PyObject *name; if (!PyUnicode_Check(arg)) { - PyErr_Format(PyExc_TypeError, "getpwnam() argument must be str, not %T", arg); + _PyArg_BadArgument("getpwnam", "argument", "str", arg); goto exit; } name = arg; @@ -71,4 +73,4 @@ pwd_getpwall(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef PWD_GETPWALL_METHODDEF #define PWD_GETPWALL_METHODDEF #endif /* !defined(PWD_GETPWALL_METHODDEF) */ -/*[clinic end generated code: output=dac88d500f6d6f49 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a8fb12939ff4ea3 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 253ad8c9e78f00..c0a5f678ad038d 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -783,9 +783,21 @@ select_epoll_register(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_pos; } - eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (eventmask == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &eventmask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional_pos: return_value = select_epoll_register_impl((pyEpoll_Object *)self, fd, eventmask); @@ -860,9 +872,21 @@ select_epoll_modify(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO if (fd < 0) { goto exit; } - eventmask = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (eventmask == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &eventmask, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = select_epoll_modify_impl((pyEpoll_Object *)self, fd, eventmask); @@ -1375,4 +1399,4 @@ select_kqueue_control(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=6fc20d78802511d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2a66dd831f22c696 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 955861b4da37d9..b0cd9e2e561640 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -660,7 +660,22 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) _PyArg_BadArgument("pthread_kill", "argument 1", "int", args[0]); goto exit; } - thread_id = PyLong_AsUnsignedLongMask(args[0]); + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &thread_id, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } + } signalnum = PyLong_AsInt(args[1]); if (signalnum == -1 && PyErr_Occurred()) { goto exit; @@ -779,4 +794,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=48bfaffeb25df5d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=37ae8ebeae4178fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 146a7e250019f0..016af258d63dea 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -1028,9 +1028,21 @@ zlib_adler32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - value = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (value == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &value, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: return_value = zlib_adler32_impl(module, &data, value); @@ -1080,13 +1092,37 @@ zlib_adler32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("adler32_combine", nargs, 3, 3)) { goto exit; } - adler1 = (unsigned int)PyLong_AsUnsignedLongMask(args[0]); - if (adler1 == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &adler1, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } - adler2 = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (adler2 == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &adler2, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } if (!PyLong_Check(args[2])) { _PyArg_BadArgument("adler32_combine", "argument 3", "int", args[2]); @@ -1137,9 +1173,21 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - value = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (value == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &value, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } skip_optional: _return_value = zlib_crc32_impl(module, &data, value); @@ -1193,13 +1241,37 @@ zlib_crc32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (!_PyArg_CheckPositional("crc32_combine", nargs, 3, 3)) { goto exit; } - crc1 = (unsigned int)PyLong_AsUnsignedLongMask(args[0]); - if (crc1 == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[0], &crc1, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } - crc2 = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (crc2 == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(args[1], &crc2, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } if (!PyLong_Check(args[2])) { _PyArg_BadArgument("crc32_combine", "argument 3", "int", args[2]); @@ -1239,4 +1311,4 @@ zlib_crc32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=3f7692eb3b5d5a0c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3054c8894aa44568 input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 90363b9dca3316..524eb54b984ca8 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,9 +1,9 @@ /* fcntl module */ -// Need limited C API version 3.13 for PyLong_AsInt() +// Need limited C API version 3.14 for PyLong_AsNativeBytes() in AC code #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +# define Py_LIMITED_API 0x030e0000 #endif #include "Python.h" diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345afbda..b89d7427e3febd 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -364,10 +364,9 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] - # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the - # presence of a pyvenv.cfg file without a 'home' key to signify the - # existence of a virtual environment โ€” we quietly ignore them. - # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! + # Search for the 'home' key in pyvenv.cfg. If a home key isn't found, + # then it means a venv is active and home is based on the venv's + # executable (if its a symlink, home is where the symlink points). for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -412,10 +411,8 @@ def search_up(prefix, *landmarks, test=isfile): if isfile(candidate): base_executable = candidate break + # home key found; stop iterating over lines break - else: - # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. - venv_prefix = None # ****************************************************************************** diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 29da9936b65504..652958618a2c4c 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -55,6 +55,11 @@ get_grp_state(PyObject *module) static struct PyModuleDef grpmodule; +/* Mutex to protect calls to getgrgid(), getgrnam(), and getgrent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex group_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -168,9 +173,15 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrgid() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrgid.html p = getgrgid(gid); #endif if (p == NULL) { +#ifndef HAVE_GETGRGID_R + PyMutex_Unlock(&group_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -185,6 +196,8 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) retval = mkgrent(module, p); #ifdef HAVE_GETGRGID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&group_db_mutex); #endif return retval; } @@ -249,9 +262,15 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrnam() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrnam.html p = getgrnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -261,6 +280,9 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkgrent(module, p); +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -285,8 +307,7 @@ grp_getgrall_impl(PyObject *module) return NULL; } - static PyMutex getgrall_mutex = {0}; - PyMutex_Lock(&getgrall_mutex); + PyMutex_Lock(&group_db_mutex); setgrent(); struct group *p; @@ -306,7 +327,7 @@ grp_getgrall_impl(PyObject *module) done: endgrent(); - PyMutex_Unlock(&getgrall_mutex); + PyMutex_Unlock(&group_db_mutex); return d; } diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 9a7e72f34a7f9d..5de5922c345047 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -2,6 +2,15 @@ #include "pycore_lock.h" // PyMutex +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + /* * Given a PyObject* obj, fill in the Py_buffer* viewp with the result * of PyObject_GetBuffer. Sets an exception and issues the erraction diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index 95e400231bb65c..b5405c99f1f8ce 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -656,7 +656,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - "unsupported hash type: %R", hash_info_ref); + HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2003546ce84cef..cc1a558001563c 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1880,8 +1880,8 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) return 0; } -static PyObject * -chain_next(PyObject *op) +static inline PyObject * +chain_next_lock_held(PyObject *op) { chainobject *lz = chainobject_CAST(op); PyObject *item; @@ -1919,6 +1919,16 @@ chain_next(PyObject *op) return NULL; } +static PyObject * +chain_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = chain_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ @@ -2086,7 +2096,7 @@ product_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -product_next(PyObject *op) +product_next_lock_held(PyObject *op) { productobject *lz = productobject_CAST(op); PyObject *pool; @@ -2172,6 +2182,16 @@ product_next(PyObject *op) return NULL; } +static PyObject * +product_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = product_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef product_methods[] = { {"__sizeof__", product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2319,7 +2339,7 @@ combinations_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -combinations_next(PyObject *op) +combinations_next_lock_held(PyObject *op) { combinationsobject *co = combinationsobject_CAST(op); PyObject *elem; @@ -2404,6 +2424,16 @@ combinations_next(PyObject *op) return NULL; } +static PyObject * +combinations_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = combinations_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef combinations_methods[] = { {"__sizeof__", combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 033de0b2907a69..7c2a421dd6a450 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1214,6 +1214,40 @@ math_floor(PyObject *module, PyObject *number) return PyLong_FromDouble(floor(x)); } +/*[clinic input] +math.fmax -> double + + x: double + y: double + / + +Return the larger of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmax_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=00692358d312fee2 input=021596c027336ffe]*/ +{ + return fmax(x, y); +} + +/*[clinic input] +math.fmin -> double + + x: double + y: double + / + +Return the smaller of two floating-point arguments. +[clinic start generated code]*/ + +static double +math_fmin_impl(PyObject *module, double x, double y) +/*[clinic end generated code: output=3d5b7826bd292dd9 input=d12e64ccc33f878a]*/ +{ + return fmin(x, y); +} + FUNC1AD(gamma, m_tgamma, "gamma($module, x, /)\n--\n\n" "Gamma function at x.", @@ -4192,7 +4226,9 @@ static PyMethodDef math_methods[] = { MATH_FACTORIAL_METHODDEF MATH_FLOOR_METHODDEF MATH_FMA_METHODDEF + MATH_FMAX_METHODDEF MATH_FMOD_METHODDEF + MATH_FMIN_METHODDEF MATH_FREXP_METHODDEF MATH_FSUM_METHODDEF {"gamma", math_gamma, METH_O, math_gamma_doc}, diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 7c4eb05488eb33..0cb4b62d734550 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,8 +23,10 @@ #endif #include <Python.h> +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #ifndef MS_WINDOWS @@ -163,8 +165,7 @@ mmap_object_dealloc(PyObject *op) Py_END_ALLOW_THREADS #endif /* UNIX */ - if (m_obj->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, m_obj->weakreflist); tp->tp_free(m_obj); Py_DECREF(tp); @@ -529,7 +530,7 @@ mmap_read_method(PyObject *op, PyObject *args) mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|n?:read", &num_bytes)) + if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) return NULL; CHECK_VALID(NULL); @@ -1723,7 +1724,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - PyObject *tagname = NULL; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1733,7 +1734,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|U?iL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1866,7 +1867,13 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL) { + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b570f81b7cf7c2..b1a80788bd8115 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3477,12 +3477,12 @@ Change the current working directory to the specified path. path may always be specified as a string. On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +If this functionality is unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=1a4a15b4d12cb15d]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=a74ceab5d72adf74]*/ { int result; @@ -5226,14 +5226,15 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) /*[clinic input] os._path_splitroot - path: path_t + path: path_t, + / Removes everything after the root on Win32. [clinic start generated code]*/ static PyObject * os__path_splitroot_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=ab7f1a88b654581c input=dc93b1d3984cffb6]*/ +/*[clinic end generated code: output=ab7f1a88b654581c input=42831e41f8458f6d]*/ { wchar_t *buffer; wchar_t *end; @@ -5535,7 +5536,8 @@ os__path_lexists_impl(PyObject *module, path_t *path) /*[clinic input] os._path_isdir -> bool - s as path: path_t(allow_fd=True, suppress_value_error=True) + path: path_t(allow_fd=True, suppress_value_error=True), + / Return true if the pathname refers to an existing directory. @@ -5543,7 +5545,7 @@ Return true if the pathname refers to an existing directory. static int os__path_isdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ +/*[clinic end generated code: output=d5786196f9e2fa7a input=0d3fd790564d244b]*/ { return _testFileType(path, PY_IFDIR); } @@ -5612,7 +5614,8 @@ os__path_isjunction_impl(PyObject *module, path_t *path) /*[clinic input] os._path_splitroot_ex - p as path: path_t(make_wide=True, nonstrict=True) + path: path_t(make_wide=True, nonstrict=True), + / Split a pathname into drive, root and tail. @@ -5621,7 +5624,7 @@ The tail contains anything after the root. static PyObject * os__path_splitroot_ex_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=4b0072b6cdf4b611 input=4556b615c7cc13f2]*/ +/*[clinic end generated code: output=4b0072b6cdf4b611 input=4ac47b394d68bd21]*/ { Py_ssize_t drvsize, rootsize; PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; @@ -16971,6 +16974,25 @@ os__emscripten_debugger_impl(PyObject *module) emscripten_debugger(); Py_RETURN_NONE; } + +EM_JS(void, emscripten_log_impl_js, (const char* arg), { + console.warn(UTF8ToString(arg)); +}); + +/*[clinic input] +os._emscripten_log + arg: str + +Log something to the JS console. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg) +/*[clinic end generated code: output=9749e5e293c42784 input=350aa1f70bc1e905]*/ +{ + emscripten_log_impl_js(arg); + Py_RETURN_NONE; +} #endif /* __EMSCRIPTEN__ */ @@ -17190,6 +17212,7 @@ static PyMethodDef posix_methods[] = { OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF OS__EMSCRIPTEN_DEBUGGER_METHODDEF + OS__EMSCRIPTEN_LOG_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index c5a8cead19a773..ac1a04a54edd60 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -1,12 +1,6 @@ /* UNIX password file access module */ -// Need limited C API version 3.13 for PyMem_RawRealloc() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif - #include "Python.h" #include "posixmodule.h" @@ -69,6 +63,11 @@ get_pwd_state(PyObject *module) static struct PyModuleDef pwdmodule; +/* Mutex to protect calls to getpwuid(), getpwnam(), and getpwent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex pwd_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -182,9 +181,15 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&pwd_db_mutex); + // The getpwuid() function is not required to be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009604499/functions/getpwuid.html p = getpwuid(uid); #endif if (p == NULL) { +#ifndef HAVE_GETPWUID_R + PyMutex_Unlock(&pwd_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -200,6 +205,8 @@ pwd_getpwuid(PyObject *module, PyObject *uidobj) retval = mkpwent(module, p); #ifdef HAVE_GETPWUID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&pwd_db_mutex); #endif return retval; } @@ -265,9 +272,15 @@ pwd_getpwnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&pwd_db_mutex); + // The getpwnam() function is not required to be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009604599/functions/getpwnam.html p = getpwnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETPWNAM_R + PyMutex_Unlock(&pwd_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -278,6 +291,9 @@ pwd_getpwnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkpwent(module, p); +#ifndef HAVE_GETPWNAM_R + PyMutex_Unlock(&pwd_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -302,12 +318,12 @@ pwd_getpwall_impl(PyObject *module) if ((d = PyList_New(0)) == NULL) return NULL; -#ifdef Py_GIL_DISABLED - static PyMutex getpwall_mutex = {0}; - PyMutex_Lock(&getpwall_mutex); -#endif + PyMutex_Lock(&pwd_db_mutex); int failure = 0; PyObject *v = NULL; + // The setpwent(), getpwent() and endpwent() functions are not required to + // be thread-safe. + // https://pubs.opengroup.org/onlinepubs/009696799/functions/setpwent.html setpwent(); while ((p = getpwent()) != NULL) { v = mkpwent(module, p); @@ -321,9 +337,7 @@ pwd_getpwall_impl(PyObject *module) done: endpwent(); -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&getpwall_mutex); -#endif + PyMutex_Unlock(&pwd_db_mutex); if (failure) { Py_XDECREF(v); Py_CLEAR(d); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index fa153d86543e99..c449dd848d10c6 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -98,7 +98,11 @@ typedef struct { #define CHARACTER_DATA_BUFFER_SIZE 8192 -typedef const void *xmlhandler; +// A generic function type for storage. +// To avoid undefined behaviors, a handler must be cast to the correct +// function type before it's called; see SETTER_WRAPPER below. +typedef void (*xmlhandler)(void); + typedef void (*xmlhandlersetter)(XML_Parser self, xmlhandler handler); struct HandlerInfo { @@ -110,9 +114,7 @@ struct HandlerInfo { static struct HandlerInfo handler_info[64]; -// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED, rather than using the exact -// handler API for each handler. -static inline void _Py_NO_SANITIZE_UNDEFINED +static inline void CALL_XML_HANDLER_SETTER(const struct HandlerInfo *handler_info, XML_Parser xml_parser, xmlhandler xml_handler) { @@ -1365,7 +1367,7 @@ xmlparse_handler_setter(PyObject *op, PyObject *v, void *closure) elaborate system of handlers and state could remove the C handler more effectively. */ if (handlernum == CharacterData && self->in_callback) { - c_handler = noop_character_data_handler; + c_handler = (xmlhandler)noop_character_data_handler; } v = NULL; } @@ -2222,13 +2224,84 @@ clear_handlers(xmlparseobject *self, int initial) } } +/* To avoid undefined behaviors, a function must be *called* via a function + * pointer of the correct type. + * So, for each `XML_Set*` function, we define a wrapper that calls `XML_Set*` + * with its argument cast to the appropriate type. + */ + +typedef void (*parser_only)(void *); +typedef int (*not_standalone)(void *); +typedef void (*parser_and_data)(void *, const XML_Char *); +typedef void (*parser_and_data_and_int)(void *, const XML_Char *, int); +typedef void (*parser_and_data_and_data)( + void *, const XML_Char *, const XML_Char *); +typedef void (*start_element)(void *, const XML_Char *, const XML_Char **); +typedef void (*element_decl)(void *, const XML_Char *, XML_Content *); +typedef void (*xml_decl)( + void *, const XML_Char *, const XML_Char *, int); +typedef void (*start_doctype_decl)( + void *, const XML_Char *, const XML_Char *, const XML_Char *, int); +typedef void (*notation_decl)( + void *, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); +typedef void (*attlist_decl)( + void *, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *, + int); +typedef void (*unparsed_entity_decl)( + void *, + const XML_Char *, const XML_Char *, + const XML_Char *, const XML_Char *, const XML_Char *); +typedef void (*entity_decl)( + void *, + const XML_Char *, int, + const XML_Char *, int, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); +typedef int (*external_entity_ref)( + XML_Parser, + const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *); + +#define SETTER_WRAPPER(NAME, TYPE) \ + static inline void \ + pyexpat_Set ## NAME (XML_Parser parser, xmlhandler handler) \ + { \ + (void)XML_Set ## NAME (parser, (TYPE)handler); \ + } + +SETTER_WRAPPER(StartElementHandler, start_element) +SETTER_WRAPPER(EndElementHandler, parser_and_data) +SETTER_WRAPPER(ProcessingInstructionHandler, parser_and_data_and_data) +SETTER_WRAPPER(CharacterDataHandler, parser_and_data_and_int) +SETTER_WRAPPER(UnparsedEntityDeclHandler, unparsed_entity_decl) +SETTER_WRAPPER(NotationDeclHandler, notation_decl) +SETTER_WRAPPER(StartNamespaceDeclHandler, parser_and_data_and_data) +SETTER_WRAPPER(EndNamespaceDeclHandler, parser_and_data) +SETTER_WRAPPER(CommentHandler, parser_and_data) +SETTER_WRAPPER(StartCdataSectionHandler, parser_only) +SETTER_WRAPPER(EndCdataSectionHandler, parser_only) +SETTER_WRAPPER(DefaultHandler, parser_and_data_and_int) +SETTER_WRAPPER(DefaultHandlerExpand, parser_and_data_and_int) +SETTER_WRAPPER(NotStandaloneHandler, not_standalone) +SETTER_WRAPPER(ExternalEntityRefHandler, external_entity_ref) +SETTER_WRAPPER(StartDoctypeDeclHandler, start_doctype_decl) +SETTER_WRAPPER(EndDoctypeDeclHandler, parser_only) +SETTER_WRAPPER(EntityDeclHandler, entity_decl) +SETTER_WRAPPER(XmlDeclHandler, xml_decl) +SETTER_WRAPPER(ElementDeclHandler, element_decl) +SETTER_WRAPPER(AttlistDeclHandler, attlist_decl) +#if XML_COMBINED_VERSION >= 19504 +SETTER_WRAPPER(SkippedEntityHandler, parser_and_data_and_int) +#endif +#undef SETTER_WRAPPER + static struct HandlerInfo handler_info[] = { // The cast to `xmlhandlersetter` is needed as the signature of XML // handler functions is not compatible with `xmlhandlersetter` since // their second parameter is narrower than a `const void *`. #define HANDLER_INFO(name) \ - {#name, (xmlhandlersetter)XML_Set##name, my_##name}, + {#name, (xmlhandlersetter)pyexpat_Set##name, (xmlhandler)my_##name}, HANDLER_INFO(StartElementHandler) HANDLER_INFO(EndElementHandler) diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index ab20fff1509dfe..5d7fd20c4e0999 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -298,7 +298,13 @@ syslog_setlogmask_impl(PyObject *module, long maskpri) return -1; } - return setlogmask(maskpri); + static PyMutex setlogmask_mutex = {0}; + PyMutex_Lock(&setlogmask_mutex); + // Linux man page (3): setlogmask() is MT-Unsafe race:LogMask. + long previous_mask = setlogmask(maskpri); + PyMutex_Unlock(&setlogmask_mutex); + + return previous_mask; } /*[clinic input] diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b5d5ca9178ebdb..bf30c06af5d8fa 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,7 +709,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +746,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +807,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +848,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, diff --git a/Objects/classobject.c b/Objects/classobject.c index 58e1d17977322e..e71f301f2efd77 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -7,6 +7,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/classobject.c.h" @@ -245,8 +246,7 @@ method_dealloc(PyObject *self) { PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); - if (im->im_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)im); + FT_CLEAR_WEAKREFS(self, im->im_weakreflist); Py_DECREF(im->im_func); Py_XDECREF(im->im_self); assert(Py_IS_TYPE(self, &PyMethod_Type)); diff --git a/Objects/clinic/interpolationobject.c.h b/Objects/clinic/interpolationobject.c.h index 7a94dabafc92f2..2030e17e49e47a 100644 --- a/Objects/clinic/interpolationobject.c.h +++ b/Objects/clinic/interpolationobject.c.h @@ -47,26 +47,31 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *value; - PyObject *expression; + PyObject *expression = &_Py_STR(empty); PyObject *conversion = Py_None; PyObject *format_spec = &_Py_STR(empty); fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } value = fastargs[0]; - if (!PyUnicode_Check(fastargs[1])) { - _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); - goto exit; - } - expression = fastargs[1]; if (!noptargs) { goto skip_optional_pos; } + if (fastargs[1]) { + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); + goto exit; + } + expression = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } if (fastargs[2]) { if (!_conversion_converter(fastargs[2], &conversion)) { goto exit; @@ -86,4 +91,4 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=599742a5ccd6f060 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2391391e2d7708c0 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 91772bc9d19098..42e021679b583f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -17,6 +17,7 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/codeobject.c.h" #include <stdbool.h> @@ -2436,9 +2437,7 @@ code_dealloc(PyObject *self) Py_XDECREF(co->_co_cached->_co_varnames); PyMem_Free(co->_co_cached); } - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, co->co_weakreflist); free_monitoring_data(co->_co_monitoring); #ifdef Py_GIL_DISABLED // The first element always points to the mutable bytecode at the end of @@ -3331,12 +3330,29 @@ _PyCodeArray_New(Py_ssize_t size) return arr; } +// Get the underlying code unit, leaving instrumentation +static _Py_CODEUNIT +deopt_code_unit(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; + int opcode = inst.op.code; + if (opcode < MIN_INSTRUMENTED_OPCODE) { + inst.op.code = _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + } + // JIT should not be enabled with free-threading + assert(inst.op.code != ENTER_EXECUTOR); + return inst; +} + static void copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) { int code_len = (int) Py_SIZE(co); for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { - dst[i] = _Py_GetBaseCodeUnit(co, i); + dst[i] = deopt_code_unit(co, i); } _PyCode_Quicken(dst, code_len, 1); } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 10c465b95ac192..d3d17e92b6d1e8 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1233,7 +1233,10 @@ static PyObject * mappingproxy_richcompare(PyObject *self, PyObject *w, int op) { mappingproxyobject *v = (mappingproxyobject *)self; - return PyObject_RichCompare(v->mapping, w, op); + if (op == Py_EQ || op == Py_NE) { + return PyObject_RichCompare(v->mapping, w, op); + } + Py_RETURN_NOTIMPLEMENTED; } static int diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6b7b150f0e2f2c..0ed52ac5e87b6e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -217,7 +217,7 @@ set_values(PyDictObject *mp, PyDictValues *values) #define LOAD_KEYS_NENTRIES(keys) _Py_atomic_load_ssize_relaxed(&keys->dk_nentries) #define INCREF_KEYS_FT(dk) dictkeys_incref(dk) -#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(_PyInterpreterState_GET(), dk, shared) +#define DECREF_KEYS_FT(dk, shared) dictkeys_decref(dk, shared) static inline void split_keys_entry_added(PyDictKeysObject *keys) { @@ -380,8 +380,7 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyInterpreterState *interp, PyDictObject *mp, - uint8_t log_newsize, int unicode); +static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyObject *dict); @@ -444,7 +443,7 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) +dictkeys_decref(PyDictKeysObject *dk, bool use_qsbr) { if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) < 0) { assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_DICT_IMMORTAL_INITIAL_REFCNT); @@ -753,7 +752,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) +new_keys_object(uint8_t log2_size, bool unicode) { Py_ssize_t usable; int log2_bytes; @@ -867,8 +866,7 @@ free_values(PyDictValues *values, bool use_qsbr) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values, +new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { assert(keys != NULL); @@ -876,7 +874,7 @@ new_dict(PyInterpreterState *interp, if (mp == NULL) { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); if (free_values_on_failure) { free_values(values, false); } @@ -894,7 +892,7 @@ new_dict(PyInterpreterState *interp, } static PyObject * -new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) +new_dict_with_shared_keys(PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); @@ -905,7 +903,7 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(interp, keys, values, 0, 1); + return new_dict(keys, values, 0, 1); } @@ -971,9 +969,8 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { - PyInterpreterState *interp = _PyInterpreterState_GET(); /* We don't incref Py_EMPTY_KEYS here because it is immortal. */ - return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1714,9 +1711,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) +insertion_resize(PyDictObject *mp, int unicode) { - return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static inline int @@ -1725,7 +1722,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, { if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { return -1; } } @@ -1823,7 +1820,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(interp, mp, 0) < 0) + if (insertion_resize(mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1838,7 +1835,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, } /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { goto Fail; } } @@ -1893,8 +1890,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, ASSERT_DICT_LOCKED(mp); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object( - interp, PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); @@ -1989,7 +1985,7 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyInterpreterState *interp, PyDictObject *mp, +dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys, *newkeys; @@ -2017,7 +2013,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, */ /* Allocate a new table. */ - newkeys = new_keys_object(interp, log2_newsize, unicode); + newkeys = new_keys_object(log2_newsize, unicode); if (newkeys == NULL) { return -1; } @@ -2060,7 +2056,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } UNLOCK_KEYS(oldkeys); set_keys(mp, newkeys); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); set_values(mp, NULL); if (oldvalues->embedded) { assert(oldvalues->embedded == 1); @@ -2141,7 +2137,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, } static PyObject * -dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) +dict_new_presized(Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -2162,17 +2158,16 @@ dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(interp, log2_newsize, unicode); + new_keys = new_keys_object(log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(interp, new_keys, NULL, 0, 0); + return new_dict(new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - return dict_new_presized(interp, minused, false); + return dict_new_presized(minused, false); } PyObject * @@ -2182,7 +2177,6 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; - PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -2192,7 +2186,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(interp, length, unicode); + PyObject *dict = dict_new_presized(length, unicode); if (dict == NULL) { return NULL; } @@ -2895,7 +2889,7 @@ clear_lock_held(PyObject *op) if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } else { n = oldkeys->dk_nentries; @@ -2909,7 +2903,7 @@ clear_lock_held(PyObject *op) set_values(mp, NULL); set_keys(mp, Py_EMPTY_KEYS); free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); + dictkeys_decref(oldkeys, false); } } ASSERT_CONSISTENT(mp); @@ -3161,7 +3155,7 @@ dict_dict_fromkeys(PyInterpreterState *interp, PyDictObject *mp, uint8_t new_size = Py_MAX( estimate_log2_keysize(PyDict_GET_SIZE(iterable)), DK_LOG_SIZE(mp->ma_keys)); - if (dictresize(interp, mp, new_size, unicode)) { + if (dictresize(mp, new_size, unicode)) { Py_DECREF(mp); return NULL; } @@ -3186,7 +3180,7 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, uint8_t new_size = Py_MAX( estimate_log2_keysize(PySet_GET_SIZE(iterable)), DK_LOG_SIZE(mp->ma_keys)); - if (dictresize(interp, mp, new_size, 0)) { + if (dictresize(mp, new_size, 0)) { Py_DECREF(mp); return NULL; } @@ -3298,11 +3292,11 @@ dict_dealloc(PyObject *self) } free_values(values, false); } - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } if (Py_IS_TYPE(mp, &PyDict_Type)) { _Py_FREELIST_FREE(dicts, mp, Py_TYPE(mp)->tp_free); @@ -3832,7 +3826,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe return -1; ensure_shared_on_resize(mp); - dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); + dictkeys_decref(mp->ma_keys, IS_DICT_SHARED(mp)); set_keys(mp, keys); STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); @@ -3851,8 +3845,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(interp, mp, - estimate_log2_keysize(mp->ma_used + other->ma_used), + if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { return -1; } @@ -4117,7 +4110,7 @@ copy_lock_held(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -4362,7 +4355,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(interp, mp, 0) < 0) { + if (insertion_resize(mp, 0) < 0) { if (result) { *result = NULL; } @@ -4386,7 +4379,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + if (insertion_resize(mp, 1) < 0) { goto error; } } @@ -4411,6 +4404,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu if (result) { *result = NULL; } + return -1; } STORE_USED(mp, mp->ma_used + 1); @@ -4554,7 +4548,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (_PyDict_HasSplitTable(self)) { - if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { + if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1) < 0) { Py_DECREF(res); return NULL; } @@ -6724,10 +6718,7 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(PyHeapTypeObject *cls) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - - PyDictKeysObject *keys = new_keys_object( - interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -6791,8 +6782,7 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) } static PyDictObject * -make_dict_from_instance_attributes(PyInterpreterState *interp, - PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -6803,7 +6793,7 @@ make_dict_from_instance_attributes(PyInterpreterState *interp, used += 1; } } - PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0); + PyDictObject *res = (PyDictObject *)new_dict(keys, values, used, 0); return res; } @@ -6817,9 +6807,8 @@ _PyObject_MaterializeManagedDict_LockHeld(PyObject *obj) PyDictValues *values = _PyObject_InlineValues(obj); PyDictObject *dict; if (values->valid) { - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); - dict = make_dict_from_instance_attributes(interp, keys, values); + dict = make_dict_from_instance_attributes(keys, values); } else { dict = (PyDictObject *)PyDict_New(); @@ -6915,7 +6904,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, if (dict == NULL) { // Make the dict but don't publish it in the object // so that no one else will see it. - dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); + dict = make_dict_from_instance_attributes(keys, values); if (dict == NULL || _PyDict_SetItem_LockHeld(dict, name, value) < 0) { Py_XDECREF(dict); @@ -7448,11 +7437,10 @@ PyObject_ClearManagedDict(PyObject *obj) "clearing an object managed dict"); /* Clear the dict */ Py_BEGIN_CRITICAL_SECTION(dict); - PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *oldkeys = dict->ma_keys; set_keys(dict, Py_EMPTY_KEYS); dict->ma_values = NULL; - dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict)); + dictkeys_decref(oldkeys, IS_DICT_SHARED(dict)); STORE_USED(dict, 0); clear_inline_values(_PyObject_InlineValues(obj)); Py_END_CRITICAL_SECTION(); @@ -7489,8 +7477,7 @@ ensure_managed_dict(PyObject *obj) goto done; } #endif - dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), - CACHED_KEYS(tp)); + dict = (PyDictObject *)new_dict_with_shared_keys(CACHED_KEYS(tp)); FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, (PyDictObject *)dict); @@ -7519,9 +7506,8 @@ ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) #endif PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); - dict = new_dict_with_shared_keys(interp, cached); + dict = new_dict_with_shared_keys(cached); } else { dict = PyDict_New(); @@ -7577,8 +7563,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - dictkeys_decref(interp, keys, false); + dictkeys_decref(keys, false); } static inline uint32_t diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 601fc69c4b1f60..97de1e06efe1b2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -913,6 +913,15 @@ static PyMethodDef framelocalsproxy_methods[] = { {NULL, NULL} /* sentinel */ }; +PyDoc_STRVAR(framelocalsproxy_doc, +"FrameLocalsProxy($frame)\n" +"--\n" +"\n" +"Create a write-through view of the locals dictionary for a frame.\n" +"\n" +" frame\n" +" the frame object to wrap."); + PyTypeObject PyFrameLocalsProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "FrameLocalsProxy", @@ -933,6 +942,7 @@ PyTypeObject PyFrameLocalsProxy_Type = { .tp_alloc = PyType_GenericAlloc, .tp_new = framelocalsproxy_new, .tp_free = PyObject_GC_Del, + .tp_doc = framelocalsproxy_doc, }; PyObject * diff --git a/Objects/funcobject.c b/Objects/funcobject.c index f8dd10a346d613..9532c21fc7082e 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -10,6 +10,7 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() static const char * @@ -1148,9 +1149,7 @@ func_dealloc(PyObject *self) return; } _PyObject_GC_UNTRACK(op); - if (op->func_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) op); - } + FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). _Py_DECREF_CODE((PyCodeObject *)op->func_code); diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 07b57f0c552ce9..3bb961aa2b619d 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -7,6 +7,7 @@ #include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> @@ -33,9 +34,7 @@ ga_dealloc(PyObject *self) gaobject *alias = (gaobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->origin); Py_XDECREF(alias->args); Py_XDECREF(alias->parameters); diff --git a/Objects/genobject.c b/Objects/genobject.c index d0cb75d2d17b07..3e7d6257006cfd 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -17,6 +17,7 @@ #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_warnings.h" // _PyErr_WarnUnawaitedCoroutine() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "opcode_ids.h" // RESUME, etc @@ -161,8 +162,7 @@ gen_dealloc(PyObject *self) _PyObject_GC_UNTRACK(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); + FT_CLEAR_WEAKREFS(self, gen->gi_weakreflist); _PyObject_GC_TRACK(self); diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index a5d407a7b0e296..b58adb693f0cae 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -54,7 +54,7 @@ typedef struct { Interpolation.__new__ as interpolation_new value: object - expression: object(subclass_of='&PyUnicode_Type') + expression: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" conversion: object(converter='_conversion_converter') = None format_spec: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" [clinic start generated code]*/ @@ -63,7 +63,7 @@ static PyObject * interpolation_new_impl(PyTypeObject *type, PyObject *value, PyObject *expression, PyObject *conversion, PyObject *format_spec) -/*[clinic end generated code: output=6488e288765bc1a9 input=d91711024068528c]*/ +/*[clinic end generated code: output=6488e288765bc1a9 input=fc5c285c1dd23d36]*/ { interpolationobject *self = PyObject_GC_New(interpolationobject, type); if (!self) { diff --git a/Objects/longobject.c b/Objects/longobject.c index 557bb6e1dd956c..581db10b54ab57 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -526,6 +526,54 @@ PyLong_FromDouble(double dval) #define PY_ABS_LONG_MIN (0-(unsigned long)LONG_MIN) #define PY_ABS_SSIZE_T_MIN (0-(size_t)PY_SSIZE_T_MIN) +static inline unsigned long +unroll_digits_ulong(PyLongObject *v, Py_ssize_t *iptr) +{ + assert(ULONG_MAX >= ((1UL << PyLong_SHIFT) - 1)); + + Py_ssize_t i = *iptr; + assert(i >= 2); + + /* unroll 1 digit */ + --i; + digit *digits = v->long_value.ob_digit; + unsigned long x = digits[i]; + +#if (ULONG_MAX >> PyLong_SHIFT) >= ((1UL << PyLong_SHIFT) - 1) + /* unroll another digit */ + x <<= PyLong_SHIFT; + --i; + x |= digits[i]; +#endif + + *iptr = i; + return x; +} + +static inline size_t +unroll_digits_size_t(PyLongObject *v, Py_ssize_t *iptr) +{ + assert(SIZE_MAX >= ((1UL << PyLong_SHIFT) - 1)); + + Py_ssize_t i = *iptr; + assert(i >= 2); + + /* unroll 1 digit */ + --i; + digit *digits = v->long_value.ob_digit; + size_t x = digits[i]; + +#if (SIZE_MAX >> PyLong_SHIFT) >= ((1 << PyLong_SHIFT) - 1) + /* unroll another digit */ + x <<= PyLong_SHIFT; + --i; + x |= digits[i]; +#endif + + *iptr = i; + return x; +} + /* Get a C long int from an int object or any object that has an __index__ method. @@ -535,13 +583,11 @@ PyLong_FromDouble(double dval) For other errors (e.g., TypeError), return -1 and set an error condition. In this case *overflow will be 0. */ - long PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) { - /* This version by Tim Peters */ + /* This version originally by Tim Peters */ PyLongObject *v; - unsigned long x, prev; long res; Py_ssize_t i; int sign; @@ -584,14 +630,14 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) res = -1; i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > (ULONG_MAX >> PyLong_SHIFT)) { *overflow = sign; goto exit; } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to long requires extra * care (see comment above). @@ -655,7 +701,6 @@ PyLong_AsInt(PyObject *obj) Py_ssize_t PyLong_AsSsize_t(PyObject *vv) { PyLongObject *v; - size_t x, prev; Py_ssize_t i; int sign; @@ -674,12 +719,13 @@ PyLong_AsSsize_t(PyObject *vv) { } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + + size_t x = unroll_digits_size_t(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) + if (x > (SIZE_MAX >> PyLong_SHIFT)) { goto overflow; + } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to a signed type requires * extra care (see comment above). @@ -705,7 +751,6 @@ unsigned long PyLong_AsUnsignedLong(PyObject *vv) { PyLongObject *v; - unsigned long x, prev; Py_ssize_t i; if (vv == NULL) { @@ -736,13 +781,13 @@ PyLong_AsUnsignedLong(PyObject *vv) return (unsigned long) -1; } i = _PyLong_DigitCount(v); - x = 0; + + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > (ULONG_MAX >> PyLong_SHIFT)) { goto overflow; } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } return x; overflow: @@ -759,7 +804,6 @@ size_t PyLong_AsSize_t(PyObject *vv) { PyLongObject *v; - size_t x, prev; Py_ssize_t i; if (vv == NULL) { @@ -781,16 +825,16 @@ PyLong_AsSize_t(PyObject *vv) return (size_t) -1; } i = _PyLong_DigitCount(v); - x = 0; + + size_t x = unroll_digits_size_t(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { - PyErr_SetString(PyExc_OverflowError, - "Python int too large to convert to C size_t"); - return (size_t) -1; + if (x > (SIZE_MAX >> PyLong_SHIFT)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C size_t"); + return (size_t) -1; + } + x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } - } return x; } @@ -801,7 +845,6 @@ static unsigned long _PyLong_AsUnsignedLongMask(PyObject *vv) { PyLongObject *v; - unsigned long x; Py_ssize_t i; if (vv == NULL || !PyLong_Check(vv)) { @@ -818,7 +861,7 @@ _PyLong_AsUnsignedLongMask(PyObject *vv) } i = _PyLong_DigitCount(v); int sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long x = unroll_digits_ulong(v, &i); while (--i >= 0) { x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } @@ -1619,7 +1662,6 @@ static unsigned long long _PyLong_AsUnsignedLongLongMask(PyObject *vv) { PyLongObject *v; - unsigned long long x; Py_ssize_t i; int sign; @@ -1637,7 +1679,7 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv) } i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long long x = unroll_digits_ulong(v, &i); while (--i >= 0) { x = (x << PyLong_SHIFT) | v->long_value.ob_digit[i]; } @@ -1683,7 +1725,6 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) { /* This version by Tim Peters */ PyLongObject *v; - unsigned long long x, prev; long long res; Py_ssize_t i; int sign; @@ -1725,15 +1766,14 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) else { i = _PyLong_DigitCount(v); sign = _PyLong_NonCompactSign(v); - x = 0; + unsigned long long x = unroll_digits_ulong(v, &i); while (--i >= 0) { - prev = x; - x = (x << PyLong_SHIFT) + v->long_value.ob_digit[i]; - if ((x >> PyLong_SHIFT) != prev) { + if (x > ULLONG_MAX >> PyLong_SHIFT) { *overflow = sign; res = -1; goto exit; } + x = (x << PyLong_SHIFT) + v->long_value.ob_digit[i]; } /* Haven't lost any bits, but casting to long requires extra * care (see comment above). diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c3dcd09ad1cdb6..e6e469ca270ac9 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -8,6 +8,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() /* undefine macro trampoline to PyCFunction_NewEx */ @@ -167,9 +168,7 @@ meth_dealloc(PyObject *self) { PyCFunctionObject *m = _PyCFunctionObject_CAST(self); PyObject_GC_UnTrack(m); - if (m->m_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*) m); - } + FT_CLEAR_WEAKREFS(self, m->m_weakreflist); // We need to access ml_flags here rather than later. // `m->m_ml` might have the same lifetime // as `m_self` when it's dynamically allocated. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ba86b41e945e9d..862395e7881870 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -13,6 +13,7 @@ #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "osdefs.h" // MAXPATHLEN @@ -826,8 +827,7 @@ module_dealloc(PyObject *self) if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } - if (m->md_weaklist != NULL) - PyObject_ClearWeakRefs((PyObject *) m); + FT_CLEAR_WEAKREFS(self, m->md_weaklist); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_free diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index 0fc2bcea4cb06e..201cb8a7df8da1 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -194,10 +194,14 @@ namespace_clear(PyObject *op) static PyObject * namespace_richcompare(PyObject *self, PyObject *other, int op) { - if (PyObject_TypeCheck(self, &_PyNamespace_Type) && - PyObject_TypeCheck(other, &_PyNamespace_Type)) + if ( + (op == Py_EQ || op == Py_NE) && + PyObject_TypeCheck(self, &_PyNamespace_Type) && + PyObject_TypeCheck(other, &_PyNamespace_Type) + ) { return PyObject_RichCompare(((_PyNamespaceObject *)self)->ns_dict, ((_PyNamespaceObject *)other)->ns_dict, op); + } Py_RETURN_NOTIMPLEMENTED; } diff --git a/Objects/object.c b/Objects/object.c index 1223983753ac46..3ed7d55593dffa 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1213,16 +1213,27 @@ PyObject_HasAttrString(PyObject *obj, const char *name) int PyObject_SetAttrString(PyObject *v, const char *name, PyObject *w) { - PyObject *s; - int res; + PyThreadState *tstate = _PyThreadState_GET(); + if (w == NULL && _PyErr_Occurred(tstate)) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetString(tstate, PyExc_SystemError, + "PyObject_SetAttrString() must not be called with NULL value " + "and an exception set"); + _PyErr_ChainExceptions1Tstate(tstate, exc); + return -1; + } - if (Py_TYPE(v)->tp_setattr != NULL) + if (Py_TYPE(v)->tp_setattr != NULL) { return (*Py_TYPE(v)->tp_setattr)(v, (char*)name, w); - s = PyUnicode_InternFromString(name); - if (s == NULL) + } + + PyObject *s = PyUnicode_InternFromString(name); + if (s == NULL) { return -1; - res = PyObject_SetAttr(v, s, w); - Py_XDECREF(s); + } + + int res = PyObject_SetAttr(v, s, w); + Py_DECREF(s); return res; } @@ -1440,6 +1451,16 @@ PyObject_HasAttr(PyObject *obj, PyObject *name) int PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) { + PyThreadState *tstate = _PyThreadState_GET(); + if (value == NULL && _PyErr_Occurred(tstate)) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetString(tstate, PyExc_SystemError, + "PyObject_SetAttr() must not be called with NULL value " + "and an exception set"); + _PyErr_ChainExceptions1Tstate(tstate, exc); + return -1; + } + PyTypeObject *tp = Py_TYPE(v); int err; @@ -1451,8 +1472,7 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) } Py_INCREF(name); - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUnicode_InternMortal(interp, &name); + _PyUnicode_InternMortal(tstate->interp, &name); if (tp->tp_setattro != NULL) { err = (*tp->tp_setattro)(v, name, value); Py_DECREF(name); diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 891f6197401503..02fcbbaa0d4536 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -473,6 +473,7 @@ Potential Optimizations #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_tuple.h" // _PyTuple_Recycle() #include <stddef.h> // offsetof() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/odictobject.c.h" @@ -1391,8 +1392,7 @@ odict_dealloc(PyObject *op) PyObject_GC_UnTrack(self); Py_XDECREF(self->od_inst_dict); - if (self->od_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); + FT_CLEAR_WEAKREFS(op, self->od_weakreflist); _odict_clear_nodes(self); PyDict_Type.tp_dealloc((PyObject *)self); diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index 3ce800de04c208..50f17687bc4365 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -1,6 +1,7 @@ /* PickleBuffer object implementation */ #include "Python.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> typedef struct { @@ -111,8 +112,7 @@ picklebuf_dealloc(PyObject *op) { PyPickleBufferObject *self = (PyPickleBufferObject*)op; PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyBuffer_Release(&self->view); Py_TYPE(self)->tp_free((PyObject *) self); } diff --git a/Objects/setobject.c b/Objects/setobject.c index 8aa6b0d180907b..6e4fc5957cad7f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -40,6 +40,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "stringlib/eq.h" // unicode_eq() #include <stddef.h> // offsetof() @@ -536,8 +537,7 @@ set_dealloc(PyObject *self) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(so); - if (so->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) so); + FT_CLEAR_WEAKREFS(self, so->weakreflist); for (entry = so->table; used > 0; entry++) { if (entry->key && entry->key != dummy) { diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 05e700b06258f0..b447865c986bef 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -595,7 +595,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, continue; } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -604,7 +604,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } @@ -668,7 +668,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -677,7 +677,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 4293a311c440f7..ac38e4de435d5d 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -30,7 +30,8 @@ templateiter_next(PyObject *op) Py_SETREF(item, PyIter_Next(self->interpolationsiter)); self->from_strings = 1; } - } else { + } + else { item = PyIter_Next(self->interpolationsiter); self->from_strings = 1; } @@ -245,54 +246,6 @@ template_iter(PyObject *op) return (PyObject *)iter; } -static PyObject * -template_strings_append_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1); - PyObject *concat = PyUnicode_Concat(string, str); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - for (Py_ssize_t i = 0; i < stringslen - 1; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - PyTuple_SET_ITEM(newstrings, stringslen - 1, concat); - - return newstrings; -} - -static PyObject * -template_strings_prepend_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, 0); - PyObject *concat = PyUnicode_Concat(str, string); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - PyTuple_SET_ITEM(newstrings, 0, concat); - for (Py_ssize_t i = 1; i < stringslen; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - - return newstrings; -} - static PyObject * template_strings_concat(PyObject *left, PyObject *right) { @@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other) return newtemplate; } -static PyObject * -template_concat_template_str(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_append_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - -static PyObject * -template_concat_str_template(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_prepend_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - PyObject * _PyTemplate_Concat(PyObject *self, PyObject *other) { if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) { return template_concat_templates((templateobject *) self, (templateobject *) other); } - else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) { - return template_concat_template_str((templateobject *) self, other); - } - else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) { - return template_concat_str_template((templateobject *) other, self); - } - else { - Py_RETURN_NOTIMPLEMENTED; - } + + PyErr_Format(PyExc_TypeError, + "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template", + other); + return NULL; } static PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 6e7471cb5941a7..d952a58d94af55 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -54,7 +54,6 @@ class object "PyObject *" "&PyBaseObject_Type" PyUnicode_CheckExact(name) && \ (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE) -#define NEXT_GLOBAL_VERSION_TAG _PyRuntime.types.next_version_tag #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag @@ -74,11 +73,11 @@ class object "PyObject *" "&PyBaseObject_Type" // while the stop-the-world mechanism is active. The slots and flags are read // in many places without holding a lock and without atomics. #define TYPE_LOCK &PyInterpreterState_Get()->types.mutex -#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK) #define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() #define BEGIN_TYPE_DICT_LOCK(d) \ - Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() @@ -226,7 +225,7 @@ type_from_ref(PyObject *ref) } -/* helpers for for static builtin types */ +/* helpers for static builtin types */ #ifndef NDEBUG static inline int @@ -266,8 +265,8 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - int64_t interp_count = - _PyRuntime.types.managed_static.types[full_index].interp_count; + int64_t interp_count = _Py_atomic_load_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count); assert((interp_count == 0) == (_PyRuntime.types.managed_static.types[full_index].type == NULL)); *p_interp_count = interp_count; @@ -344,7 +343,7 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; assert((initial == 1) == - (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) == 0)); (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); @@ -393,7 +392,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, : &(interp->types.for_extensions.initialized[index]); assert(state != NULL); - assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) > 0); assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); @@ -403,7 +402,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); if (final) { - assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + assert(!_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count)); _PyRuntime.types.managed_static.types[full_index].type = NULL; managed_static_type_index_clear(self); @@ -1359,6 +1358,19 @@ _PyType_LookupByVersion(unsigned int version) #error "_Py_ATTR_CACHE_UNUSED must be bigger than max" #endif +static inline unsigned int +next_global_version_tag(void) +{ + unsigned int old; + do { + old = _Py_atomic_load_uint_relaxed(&_PyRuntime.types.next_version_tag); + if (old >= _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + return 0; + } + } while (!_Py_atomic_compare_exchange_uint(&_PyRuntime.types.next_version_tag, &old, old + 1)); + return old + 1; +} + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -1389,11 +1401,12 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ - if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + unsigned int next_version_tag = next_global_version_tag(); + if (next_version_tag == 0) { /* We have run out of version numbers */ return 0; } - set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); + set_version_unlocked(type, next_version_tag); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -9007,7 +9020,11 @@ type_ready_set_new(PyTypeObject *type, int initial) && base == &PyBaseObject_Type && !(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + if (initial) { + type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + } else { + assert(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION); + } } if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { @@ -9021,13 +9038,17 @@ type_ready_set_new(PyTypeObject *type, int initial) } } else { - // tp_new is NULL: inherit tp_new from base - type->tp_new = base->tp_new; + if (initial) { + // tp_new is NULL: inherit tp_new from base + type->tp_new = base->tp_new; + } } } else { // Py_TPFLAGS_DISALLOW_INSTANTIATION sets tp_new to NULL - type->tp_new = NULL; + if (initial) { + type->tp_new = NULL; + } } return 0; } @@ -9160,7 +9181,12 @@ type_ready(PyTypeObject *type, int initial) } /* All done -- set the ready flag */ - type_add_flags(type, Py_TPFLAGS_READY); + if (initial) { + type_add_flags(type, Py_TPFLAGS_READY); + } else { + assert(type->tp_flags & Py_TPFLAGS_READY); + } + stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -9209,15 +9235,16 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(initial); + if (initial) { + assert((self->tp_flags & Py_TPFLAGS_READY) == 0); type_add_flags(self, _Py_TPFLAGS_STATIC_BUILTIN); type_add_flags(self, Py_TPFLAGS_IMMUTABLETYPE); - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); if (self->tp_version_tag == 0) { - _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + unsigned int next_version_tag = next_global_version_tag(); + assert(next_version_tag != 0); + _PyType_SetVersion(self, next_version_tag); } } else { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5c2308a012142a..8df7a48284dccd 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_template.h" // _PyTemplate_Concat() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -11610,16 +11609,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right) return NULL; if (!PyUnicode_Check(right)) { - if (_PyTemplate_CheckExact(right)) { - // str + tstring is implemented in the tstring type - return _PyTemplate_Concat(left, right); - } - else { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - Py_TYPE(right)->tp_name); - return NULL; - } + PyErr_Format(PyExc_TypeError, + "can only concatenate str (not \"%.200s\") to str", + Py_TYPE(right)->tp_name); + return NULL; } /* Shortcuts */ diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 00ca5b9bf80341..2206ed80ef03fd 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -4,6 +4,7 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString #include "pycore_unionobject.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef struct { @@ -21,9 +22,7 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->args); Py_XDECREF(alias->hashable_args); diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index a77d0855af293f..647aadfa46bbed 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -690,9 +690,21 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) PyObject *return_value = NULL; unsigned int mode; - mode = (unsigned int)PyLong_AsUnsignedLongMask(arg); - if (mode == (unsigned int)-1 && PyErr_Occurred()) { - goto exit; + { + Py_ssize_t _bytes = PyLong_AsNativeBytes(arg, &mode, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) { + goto exit; + } + if ((size_t)_bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + goto exit; + } + } } return_value = msvcrt_SetErrorMode_impl(module, mode); @@ -731,4 +743,4 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_GETERRORMODE_METHODDEF #define MSVCRT_GETERRORMODE_METHODDEF #endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ -/*[clinic end generated code: output=692c6f52bb9193ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f67eaf745685429d input=a9049054013a1b77]*/ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index efff6a58d895cb..5ceddf759b8f3b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -106,6 +106,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\atexitmodule.c" /> + <ClCompile Include="..\Modules\_datetimemodule.c" /> <ClCompile Include="..\Modules\faulthandler.c" /> <ClCompile Include="..\Modules\gcmodule.c" /> <ClCompile Include="..\Modules\getbuildinfo.c" /> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b911c9385634d7..517103acea8d8e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -165,7 +165,7 @@ <ClInclude Include="..\Include\cpython\import.h" /> <ClInclude Include="..\Include\cpython\initconfig.h" /> <ClInclude Include="..\Include\cpython\listobject.h" /> - <ClInclude Include="..\Include\cpython\lock.h" /> + <ClInclude Include="..\Include\cpython\pylock.h" /> <ClInclude Include="..\Include\cpython\longintrepr.h" /> <ClInclude Include="..\Include\cpython\longobject.h" /> <ClInclude Include="..\Include\cpython\memoryobject.h" /> @@ -332,7 +332,7 @@ <ClInclude Include="..\Include\intrcheck.h" /> <ClInclude Include="..\Include\iterobject.h" /> <ClInclude Include="..\Include\listobject.h" /> - <ClInclude Include="..\Include\lock.h" /> + <ClInclude Include="..\Include\pylock.h" /> <ClInclude Include="..\Include\longobject.h" /> <ClInclude Include="..\Include\marshal.h" /> <ClInclude Include="..\Include\memoryobject.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0e6d42cc959ba5..e9eedfd1312fae 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -120,7 +120,7 @@ <ClInclude Include="..\Include\listobject.h"> <Filter>Include</Filter> </ClInclude> - <ClInclude Include="..\Include\lock.h"> + <ClInclude Include="..\Include\pylock.h"> <Filter>Include</Filter> </ClInclude> <ClInclude Include="..\Include\longobject.h"> @@ -414,7 +414,7 @@ <ClInclude Include="..\Include\cpython\listobject.h"> <Filter>Include\cpython</Filter> </ClInclude> - <ClInclude Include="..\Include\cpython\lock.h"> + <ClInclude Include="..\Include\cpython\pylock.h"> <Filter>Include</Filter> </ClInclude> <ClInclude Include="..\Include\cpython\longintrepr.h"> diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 0d362bf7a9111a..ebc94715b6f361 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings, return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena); } -static expr_ty -_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings, +expr_ty +_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { @@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t len = asdl_seq_LEN(strings); assert(len > 0); - int t_string_found = 0; int f_string_found = 0; int unicode_string_found = 0; int bytes_found = 0; @@ -1873,7 +1872,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, f_string_found = 1; break; case TemplateStr_kind: - t_string_found = 1; + // python.gram handles this; we should never get here + assert(0); break; default: f_string_found = 1; @@ -1882,13 +1882,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } // Cannot mix unicode and bytes - if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) { + if ((unicode_string_found || f_string_found) && bytes_found) { RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals"); return NULL; } // If it's only bytes or only unicode string, do a simple concat - if (!f_string_found && !t_string_found) { + if (!f_string_found) { if (len == 1) { return asdl_seq_GET(strings, 0); } @@ -1902,11 +1902,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } } - if (t_string_found) { - return _build_concatenated_template_str(p, strings, lineno, - col_offset, end_lineno, end_col_offset, arena); - } - return _build_concatenated_joined_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 0a078dd594148c..81363cf8e810fe 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -121,38 +121,88 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } PyObject *res = NULL; - // Check if there is a # character in the expression + // Look for a # character outside of string literals int hash_detected = 0; + int in_string = 0; + char quote_char = 0; + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { + char ch = tok_mode->last_expr_buffer[i]; + + // Skip escaped characters + if (ch == '\\') { + i++; + continue; + } + + // Handle quotes + if (ch == '"' || ch == '\'') { + // The following if/else block works becase there is an off number + // of quotes in STRING tokens and the lexer only ever reaches this + // function with valid STRING tokens. + // For example: """hello""" + // First quote: in_string = 1 + // Second quote: in_string = 0 + // Third quote: in_string = 1 + if (!in_string) { + in_string = 1; + quote_char = ch; + } + else if (ch == quote_char) { + in_string = 0; + } + continue; + } + + // Check for # outside strings + if (ch == '#' && !in_string) { hash_detected = 1; break; } } - + // If we found a # character in the expression, we need to handle comments if (hash_detected) { - Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); + // Allocate buffer for processed result + char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); if (!result) { return -1; } - Py_ssize_t i = 0; - Py_ssize_t j = 0; + Py_ssize_t i = 0; // Input position + Py_ssize_t j = 0; // Output position + in_string = 0; // Whether we're in a string + quote_char = 0; // Current string quote char - for (i = 0, j = 0; i < input_length; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { - // Skip characters until newline or end of string - while (i < input_length && tok_mode->last_expr_buffer[i] != '\0') { - if (tok_mode->last_expr_buffer[i] == '\n') { - result[j++] = tok_mode->last_expr_buffer[i]; - break; - } + // Process each character + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + char ch = tok_mode->last_expr_buffer[i]; + + // Handle string quotes + if (ch == '"' || ch == '\'') { + // See comment above to understand this part + if (!in_string) { + in_string = 1; + quote_char = ch; + } else if (ch == quote_char) { + in_string = 0; + } + result[j++] = ch; + } + // Skip comments + else if (ch == '#' && !in_string) { + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && + tok_mode->last_expr_buffer[i] != '\n') { i++; } - } else { - result[j++] = tok_mode->last_expr_buffer[i]; + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = '\n'; + } + } + // Copy other chars + else { + result[j++] = ch; } + i++; } result[j] = '\0'; // Null-terminate the result string @@ -164,11 +214,9 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { tok_mode->last_expr_size - tok_mode->last_expr_end, NULL ); - } - - if (!res) { + if (!res) { return -1; } token->metadata = res; diff --git a/Parser/parser.c b/Parser/parser.c index ee0aeb4e1875cb..de5cdc9b6f7f34 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -21,59 +21,59 @@ static KeywordToken *reserved_keywords[] = { (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) {{NULL, -1}}, (KeywordToken[]) { - {"if", 686}, - {"as", 684}, - {"in", 699}, - {"or", 588}, - {"is", 596}, + {"if", 687}, + {"as", 685}, + {"in", 700}, + {"or", 589}, + {"is", 597}, {NULL, -1}, }, (KeywordToken[]) { - {"del", 629}, - {"def", 703}, - {"for", 698}, - {"try", 660}, - {"and", 589}, - {"not", 707}, + {"del", 630}, + {"def", 704}, + {"for", 699}, + {"try", 661}, + {"and", 590}, + {"not", 708}, {NULL, -1}, }, (KeywordToken[]) { - {"from", 637}, - {"pass", 526}, - {"with", 651}, - {"elif", 691}, - {"else", 690}, - {"None", 623}, - {"True", 622}, + {"from", 638}, + {"pass", 527}, + {"with", 652}, + {"elif", 692}, + {"else", 691}, + {"None", 624}, + {"True", 623}, {NULL, -1}, }, (KeywordToken[]) { - {"raise", 627}, - {"yield", 587}, - {"break", 527}, - {"async", 702}, - {"class", 705}, - {"while", 693}, - {"False", 624}, - {"await", 597}, + {"raise", 628}, + {"yield", 588}, + {"break", 528}, + {"async", 703}, + {"class", 706}, + {"while", 694}, + {"False", 625}, + {"await", 598}, {NULL, -1}, }, (KeywordToken[]) { {"return", 522}, - {"import", 638}, - {"assert", 532}, - {"global", 529}, - {"except", 681}, - {"lambda", 621}, + {"import", 639}, + {"assert", 533}, + {"global", 530}, + {"except", 682}, + {"lambda", 622}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 677}, + {"finally", 678}, {NULL, -1}, }, (KeywordToken[]) { - {"continue", 528}, - {"nonlocal", 530}, + {"continue", 529}, + {"nonlocal", 531}, {NULL, -1}, }, }; @@ -349,185 +349,186 @@ static char *soft_keywords[] = { #define invalid_fstring_conversion_character_type 1262 #define invalid_tstring_replacement_field_type 1263 #define invalid_tstring_conversion_character_type 1264 -#define invalid_arithmetic_type 1265 -#define invalid_factor_type 1266 -#define invalid_type_params_type 1267 -#define _loop0_1_type 1268 -#define _loop1_2_type 1269 -#define _loop0_3_type 1270 -#define _gather_4_type 1271 -#define _tmp_5_type 1272 -#define _tmp_6_type 1273 -#define _tmp_7_type 1274 -#define _tmp_8_type 1275 -#define _tmp_9_type 1276 -#define _tmp_10_type 1277 -#define _tmp_11_type 1278 -#define _loop1_12_type 1279 -#define _tmp_13_type 1280 -#define _loop0_14_type 1281 -#define _gather_15_type 1282 -#define _tmp_16_type 1283 -#define _tmp_17_type 1284 -#define _loop0_18_type 1285 -#define _loop1_19_type 1286 -#define _loop0_20_type 1287 -#define _gather_21_type 1288 -#define _tmp_22_type 1289 -#define _loop0_23_type 1290 -#define _gather_24_type 1291 -#define _loop1_25_type 1292 -#define _tmp_26_type 1293 -#define _tmp_27_type 1294 -#define _loop0_28_type 1295 -#define _loop0_29_type 1296 -#define _loop1_30_type 1297 -#define _loop1_31_type 1298 -#define _loop0_32_type 1299 -#define _loop1_33_type 1300 -#define _loop0_34_type 1301 -#define _gather_35_type 1302 -#define _tmp_36_type 1303 -#define _loop1_37_type 1304 -#define _loop1_38_type 1305 -#define _loop1_39_type 1306 -#define _loop0_40_type 1307 -#define _gather_41_type 1308 -#define _tmp_42_type 1309 -#define _tmp_43_type 1310 -#define _tmp_44_type 1311 -#define _loop0_45_type 1312 -#define _gather_46_type 1313 -#define _loop0_47_type 1314 -#define _gather_48_type 1315 -#define _tmp_49_type 1316 -#define _loop0_50_type 1317 -#define _gather_51_type 1318 -#define _loop0_52_type 1319 -#define _gather_53_type 1320 -#define _loop0_54_type 1321 -#define _gather_55_type 1322 -#define _loop1_56_type 1323 -#define _loop1_57_type 1324 -#define _loop0_58_type 1325 -#define _gather_59_type 1326 -#define _loop1_60_type 1327 -#define _loop1_61_type 1328 -#define _loop1_62_type 1329 -#define _tmp_63_type 1330 -#define _loop0_64_type 1331 -#define _gather_65_type 1332 -#define _tmp_66_type 1333 -#define _tmp_67_type 1334 -#define _tmp_68_type 1335 -#define _tmp_69_type 1336 -#define _tmp_70_type 1337 -#define _loop0_71_type 1338 -#define _loop0_72_type 1339 -#define _loop1_73_type 1340 -#define _loop1_74_type 1341 -#define _loop0_75_type 1342 -#define _loop1_76_type 1343 -#define _loop0_77_type 1344 -#define _loop0_78_type 1345 -#define _loop0_79_type 1346 -#define _loop0_80_type 1347 -#define _loop1_81_type 1348 -#define _tmp_82_type 1349 -#define _loop0_83_type 1350 -#define _gather_84_type 1351 -#define _loop1_85_type 1352 -#define _loop0_86_type 1353 -#define _tmp_87_type 1354 -#define _loop0_88_type 1355 -#define _gather_89_type 1356 -#define _tmp_90_type 1357 -#define _loop0_91_type 1358 -#define _gather_92_type 1359 -#define _loop0_93_type 1360 -#define _gather_94_type 1361 -#define _loop0_95_type 1362 -#define _loop0_96_type 1363 -#define _gather_97_type 1364 -#define _loop1_98_type 1365 -#define _tmp_99_type 1366 -#define _loop0_100_type 1367 -#define _gather_101_type 1368 -#define _loop0_102_type 1369 -#define _gather_103_type 1370 -#define _tmp_104_type 1371 -#define _tmp_105_type 1372 -#define _loop0_106_type 1373 -#define _gather_107_type 1374 -#define _tmp_108_type 1375 -#define _tmp_109_type 1376 -#define _tmp_110_type 1377 -#define _tmp_111_type 1378 -#define _tmp_112_type 1379 -#define _loop1_113_type 1380 -#define _tmp_114_type 1381 -#define _tmp_115_type 1382 -#define _tmp_116_type 1383 -#define _tmp_117_type 1384 -#define _tmp_118_type 1385 -#define _loop0_119_type 1386 -#define _loop0_120_type 1387 -#define _tmp_121_type 1388 -#define _tmp_122_type 1389 -#define _tmp_123_type 1390 -#define _tmp_124_type 1391 -#define _tmp_125_type 1392 -#define _tmp_126_type 1393 -#define _tmp_127_type 1394 -#define _tmp_128_type 1395 -#define _tmp_129_type 1396 -#define _loop0_130_type 1397 -#define _gather_131_type 1398 -#define _tmp_132_type 1399 -#define _tmp_133_type 1400 -#define _tmp_134_type 1401 -#define _tmp_135_type 1402 -#define _loop0_136_type 1403 -#define _gather_137_type 1404 -#define _tmp_138_type 1405 -#define _loop0_139_type 1406 -#define _gather_140_type 1407 -#define _loop0_141_type 1408 -#define _gather_142_type 1409 -#define _tmp_143_type 1410 -#define _loop0_144_type 1411 -#define _tmp_145_type 1412 -#define _tmp_146_type 1413 -#define _tmp_147_type 1414 -#define _tmp_148_type 1415 -#define _tmp_149_type 1416 -#define _tmp_150_type 1417 -#define _tmp_151_type 1418 -#define _tmp_152_type 1419 -#define _tmp_153_type 1420 -#define _tmp_154_type 1421 -#define _tmp_155_type 1422 -#define _tmp_156_type 1423 -#define _tmp_157_type 1424 -#define _tmp_158_type 1425 -#define _tmp_159_type 1426 -#define _tmp_160_type 1427 -#define _tmp_161_type 1428 -#define _tmp_162_type 1429 -#define _tmp_163_type 1430 -#define _tmp_164_type 1431 -#define _tmp_165_type 1432 -#define _tmp_166_type 1433 -#define _tmp_167_type 1434 -#define _tmp_168_type 1435 -#define _tmp_169_type 1436 -#define _tmp_170_type 1437 -#define _loop0_171_type 1438 -#define _tmp_172_type 1439 -#define _tmp_173_type 1440 -#define _tmp_174_type 1441 -#define _tmp_175_type 1442 -#define _tmp_176_type 1443 +#define invalid_string_tstring_concat_type 1265 +#define invalid_arithmetic_type 1266 +#define invalid_factor_type 1267 +#define invalid_type_params_type 1268 +#define _loop0_1_type 1269 +#define _loop1_2_type 1270 +#define _loop0_3_type 1271 +#define _gather_4_type 1272 +#define _tmp_5_type 1273 +#define _tmp_6_type 1274 +#define _tmp_7_type 1275 +#define _tmp_8_type 1276 +#define _tmp_9_type 1277 +#define _tmp_10_type 1278 +#define _tmp_11_type 1279 +#define _loop1_12_type 1280 +#define _loop0_13_type 1281 +#define _gather_14_type 1282 +#define _tmp_15_type 1283 +#define _tmp_16_type 1284 +#define _loop0_17_type 1285 +#define _loop1_18_type 1286 +#define _loop0_19_type 1287 +#define _gather_20_type 1288 +#define _tmp_21_type 1289 +#define _loop0_22_type 1290 +#define _gather_23_type 1291 +#define _loop1_24_type 1292 +#define _tmp_25_type 1293 +#define _tmp_26_type 1294 +#define _loop0_27_type 1295 +#define _loop0_28_type 1296 +#define _loop1_29_type 1297 +#define _loop1_30_type 1298 +#define _loop0_31_type 1299 +#define _loop1_32_type 1300 +#define _loop0_33_type 1301 +#define _gather_34_type 1302 +#define _tmp_35_type 1303 +#define _loop1_36_type 1304 +#define _loop1_37_type 1305 +#define _loop1_38_type 1306 +#define _loop0_39_type 1307 +#define _gather_40_type 1308 +#define _tmp_41_type 1309 +#define _tmp_42_type 1310 +#define _tmp_43_type 1311 +#define _loop0_44_type 1312 +#define _gather_45_type 1313 +#define _loop0_46_type 1314 +#define _gather_47_type 1315 +#define _tmp_48_type 1316 +#define _loop0_49_type 1317 +#define _gather_50_type 1318 +#define _loop0_51_type 1319 +#define _gather_52_type 1320 +#define _loop0_53_type 1321 +#define _gather_54_type 1322 +#define _loop1_55_type 1323 +#define _loop1_56_type 1324 +#define _loop0_57_type 1325 +#define _gather_58_type 1326 +#define _loop1_59_type 1327 +#define _loop1_60_type 1328 +#define _loop1_61_type 1329 +#define _tmp_62_type 1330 +#define _loop0_63_type 1331 +#define _gather_64_type 1332 +#define _tmp_65_type 1333 +#define _tmp_66_type 1334 +#define _tmp_67_type 1335 +#define _tmp_68_type 1336 +#define _tmp_69_type 1337 +#define _loop0_70_type 1338 +#define _loop0_71_type 1339 +#define _loop1_72_type 1340 +#define _loop1_73_type 1341 +#define _loop0_74_type 1342 +#define _loop1_75_type 1343 +#define _loop0_76_type 1344 +#define _loop0_77_type 1345 +#define _loop0_78_type 1346 +#define _loop0_79_type 1347 +#define _loop1_80_type 1348 +#define _loop1_81_type 1349 +#define _tmp_82_type 1350 +#define _loop0_83_type 1351 +#define _gather_84_type 1352 +#define _loop1_85_type 1353 +#define _loop0_86_type 1354 +#define _tmp_87_type 1355 +#define _loop0_88_type 1356 +#define _gather_89_type 1357 +#define _tmp_90_type 1358 +#define _loop0_91_type 1359 +#define _gather_92_type 1360 +#define _loop0_93_type 1361 +#define _gather_94_type 1362 +#define _loop0_95_type 1363 +#define _loop0_96_type 1364 +#define _gather_97_type 1365 +#define _loop1_98_type 1366 +#define _tmp_99_type 1367 +#define _loop0_100_type 1368 +#define _gather_101_type 1369 +#define _loop0_102_type 1370 +#define _gather_103_type 1371 +#define _tmp_104_type 1372 +#define _tmp_105_type 1373 +#define _loop0_106_type 1374 +#define _gather_107_type 1375 +#define _tmp_108_type 1376 +#define _tmp_109_type 1377 +#define _tmp_110_type 1378 +#define _tmp_111_type 1379 +#define _tmp_112_type 1380 +#define _loop1_113_type 1381 +#define _tmp_114_type 1382 +#define _tmp_115_type 1383 +#define _tmp_116_type 1384 +#define _tmp_117_type 1385 +#define _tmp_118_type 1386 +#define _loop0_119_type 1387 +#define _loop0_120_type 1388 +#define _tmp_121_type 1389 +#define _tmp_122_type 1390 +#define _tmp_123_type 1391 +#define _tmp_124_type 1392 +#define _tmp_125_type 1393 +#define _tmp_126_type 1394 +#define _tmp_127_type 1395 +#define _tmp_128_type 1396 +#define _tmp_129_type 1397 +#define _loop0_130_type 1398 +#define _gather_131_type 1399 +#define _tmp_132_type 1400 +#define _tmp_133_type 1401 +#define _tmp_134_type 1402 +#define _tmp_135_type 1403 +#define _loop0_136_type 1404 +#define _gather_137_type 1405 +#define _tmp_138_type 1406 +#define _loop0_139_type 1407 +#define _gather_140_type 1408 +#define _loop0_141_type 1409 +#define _gather_142_type 1410 +#define _tmp_143_type 1411 +#define _loop0_144_type 1412 +#define _tmp_145_type 1413 +#define _tmp_146_type 1414 +#define _tmp_147_type 1415 +#define _tmp_148_type 1416 +#define _tmp_149_type 1417 +#define _tmp_150_type 1418 +#define _tmp_151_type 1419 +#define _tmp_152_type 1420 +#define _tmp_153_type 1421 +#define _tmp_154_type 1422 +#define _tmp_155_type 1423 +#define _tmp_156_type 1424 +#define _tmp_157_type 1425 +#define _tmp_158_type 1426 +#define _tmp_159_type 1427 +#define _tmp_160_type 1428 +#define _tmp_161_type 1429 +#define _tmp_162_type 1430 +#define _tmp_163_type 1431 +#define _tmp_164_type 1432 +#define _tmp_165_type 1433 +#define _tmp_166_type 1434 +#define _tmp_167_type 1435 +#define _tmp_168_type 1436 +#define _tmp_169_type 1437 +#define _tmp_170_type 1438 +#define _loop0_171_type 1439 +#define _tmp_172_type 1440 +#define _tmp_173_type 1441 +#define _tmp_174_type 1442 +#define _tmp_175_type 1443 +#define _tmp_176_type 1444 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -794,6 +795,7 @@ static void *invalid_fstring_replacement_field_rule(Parser *p); static void *invalid_fstring_conversion_character_rule(Parser *p); static void *invalid_tstring_replacement_field_rule(Parser *p); static void *invalid_tstring_conversion_character_rule(Parser *p); +static void *invalid_string_tstring_concat_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); @@ -809,74 +811,74 @@ static void *_tmp_9_rule(Parser *p); static void *_tmp_10_rule(Parser *p); static void *_tmp_11_rule(Parser *p); static asdl_seq *_loop1_12_rule(Parser *p); -static void *_tmp_13_rule(Parser *p); -static asdl_seq *_loop0_14_rule(Parser *p); -static asdl_seq *_gather_15_rule(Parser *p); +static asdl_seq *_loop0_13_rule(Parser *p); +static asdl_seq *_gather_14_rule(Parser *p); +static void *_tmp_15_rule(Parser *p); static void *_tmp_16_rule(Parser *p); -static void *_tmp_17_rule(Parser *p); -static asdl_seq *_loop0_18_rule(Parser *p); -static asdl_seq *_loop1_19_rule(Parser *p); -static asdl_seq *_loop0_20_rule(Parser *p); -static asdl_seq *_gather_21_rule(Parser *p); -static void *_tmp_22_rule(Parser *p); -static asdl_seq *_loop0_23_rule(Parser *p); -static asdl_seq *_gather_24_rule(Parser *p); -static asdl_seq *_loop1_25_rule(Parser *p); +static asdl_seq *_loop0_17_rule(Parser *p); +static asdl_seq *_loop1_18_rule(Parser *p); +static asdl_seq *_loop0_19_rule(Parser *p); +static asdl_seq *_gather_20_rule(Parser *p); +static void *_tmp_21_rule(Parser *p); +static asdl_seq *_loop0_22_rule(Parser *p); +static asdl_seq *_gather_23_rule(Parser *p); +static asdl_seq *_loop1_24_rule(Parser *p); +static void *_tmp_25_rule(Parser *p); static void *_tmp_26_rule(Parser *p); -static void *_tmp_27_rule(Parser *p); +static asdl_seq *_loop0_27_rule(Parser *p); static asdl_seq *_loop0_28_rule(Parser *p); -static asdl_seq *_loop0_29_rule(Parser *p); +static asdl_seq *_loop1_29_rule(Parser *p); static asdl_seq *_loop1_30_rule(Parser *p); -static asdl_seq *_loop1_31_rule(Parser *p); -static asdl_seq *_loop0_32_rule(Parser *p); -static asdl_seq *_loop1_33_rule(Parser *p); -static asdl_seq *_loop0_34_rule(Parser *p); -static asdl_seq *_gather_35_rule(Parser *p); -static void *_tmp_36_rule(Parser *p); +static asdl_seq *_loop0_31_rule(Parser *p); +static asdl_seq *_loop1_32_rule(Parser *p); +static asdl_seq *_loop0_33_rule(Parser *p); +static asdl_seq *_gather_34_rule(Parser *p); +static void *_tmp_35_rule(Parser *p); +static asdl_seq *_loop1_36_rule(Parser *p); static asdl_seq *_loop1_37_rule(Parser *p); static asdl_seq *_loop1_38_rule(Parser *p); -static asdl_seq *_loop1_39_rule(Parser *p); -static asdl_seq *_loop0_40_rule(Parser *p); -static asdl_seq *_gather_41_rule(Parser *p); +static asdl_seq *_loop0_39_rule(Parser *p); +static asdl_seq *_gather_40_rule(Parser *p); +static void *_tmp_41_rule(Parser *p); static void *_tmp_42_rule(Parser *p); static void *_tmp_43_rule(Parser *p); -static void *_tmp_44_rule(Parser *p); -static asdl_seq *_loop0_45_rule(Parser *p); -static asdl_seq *_gather_46_rule(Parser *p); -static asdl_seq *_loop0_47_rule(Parser *p); -static asdl_seq *_gather_48_rule(Parser *p); -static void *_tmp_49_rule(Parser *p); -static asdl_seq *_loop0_50_rule(Parser *p); -static asdl_seq *_gather_51_rule(Parser *p); -static asdl_seq *_loop0_52_rule(Parser *p); -static asdl_seq *_gather_53_rule(Parser *p); -static asdl_seq *_loop0_54_rule(Parser *p); -static asdl_seq *_gather_55_rule(Parser *p); +static asdl_seq *_loop0_44_rule(Parser *p); +static asdl_seq *_gather_45_rule(Parser *p); +static asdl_seq *_loop0_46_rule(Parser *p); +static asdl_seq *_gather_47_rule(Parser *p); +static void *_tmp_48_rule(Parser *p); +static asdl_seq *_loop0_49_rule(Parser *p); +static asdl_seq *_gather_50_rule(Parser *p); +static asdl_seq *_loop0_51_rule(Parser *p); +static asdl_seq *_gather_52_rule(Parser *p); +static asdl_seq *_loop0_53_rule(Parser *p); +static asdl_seq *_gather_54_rule(Parser *p); +static asdl_seq *_loop1_55_rule(Parser *p); static asdl_seq *_loop1_56_rule(Parser *p); -static asdl_seq *_loop1_57_rule(Parser *p); -static asdl_seq *_loop0_58_rule(Parser *p); -static asdl_seq *_gather_59_rule(Parser *p); +static asdl_seq *_loop0_57_rule(Parser *p); +static asdl_seq *_gather_58_rule(Parser *p); +static asdl_seq *_loop1_59_rule(Parser *p); static asdl_seq *_loop1_60_rule(Parser *p); static asdl_seq *_loop1_61_rule(Parser *p); -static asdl_seq *_loop1_62_rule(Parser *p); -static void *_tmp_63_rule(Parser *p); -static asdl_seq *_loop0_64_rule(Parser *p); -static asdl_seq *_gather_65_rule(Parser *p); +static void *_tmp_62_rule(Parser *p); +static asdl_seq *_loop0_63_rule(Parser *p); +static asdl_seq *_gather_64_rule(Parser *p); +static void *_tmp_65_rule(Parser *p); static void *_tmp_66_rule(Parser *p); static void *_tmp_67_rule(Parser *p); static void *_tmp_68_rule(Parser *p); static void *_tmp_69_rule(Parser *p); -static void *_tmp_70_rule(Parser *p); +static asdl_seq *_loop0_70_rule(Parser *p); static asdl_seq *_loop0_71_rule(Parser *p); -static asdl_seq *_loop0_72_rule(Parser *p); +static asdl_seq *_loop1_72_rule(Parser *p); static asdl_seq *_loop1_73_rule(Parser *p); -static asdl_seq *_loop1_74_rule(Parser *p); -static asdl_seq *_loop0_75_rule(Parser *p); -static asdl_seq *_loop1_76_rule(Parser *p); +static asdl_seq *_loop0_74_rule(Parser *p); +static asdl_seq *_loop1_75_rule(Parser *p); +static asdl_seq *_loop0_76_rule(Parser *p); static asdl_seq *_loop0_77_rule(Parser *p); static asdl_seq *_loop0_78_rule(Parser *p); static asdl_seq *_loop0_79_rule(Parser *p); -static asdl_seq *_loop0_80_rule(Parser *p); +static asdl_seq *_loop1_80_rule(Parser *p); static asdl_seq *_loop1_81_rule(Parser *p); static void *_tmp_82_rule(Parser *p); static asdl_seq *_loop0_83_rule(Parser *p); @@ -1700,7 +1702,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'raise' raise_stmt")); stmt_ty raise_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 627) // token='raise' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 628) // token='raise' && (raise_stmt_var = raise_stmt_rule(p)) // raise_stmt ) @@ -1721,7 +1723,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'pass' pass_stmt")); stmt_ty pass_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 526) // token='pass' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 527) // token='pass' && (pass_stmt_var = pass_stmt_rule(p)) // pass_stmt ) @@ -1742,7 +1744,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'del' del_stmt")); stmt_ty del_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 629) // token='del' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 630) // token='del' && (del_stmt_var = del_stmt_rule(p)) // del_stmt ) @@ -1763,7 +1765,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'yield' yield_stmt")); stmt_ty yield_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 587) // token='yield' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 588) // token='yield' && (yield_stmt_var = yield_stmt_rule(p)) // yield_stmt ) @@ -1784,7 +1786,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'assert' assert_stmt")); stmt_ty assert_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 532) // token='assert' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 533) // token='assert' && (assert_stmt_var = assert_stmt_rule(p)) // assert_stmt ) @@ -1805,7 +1807,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'break' break_stmt")); stmt_ty break_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 527) // token='break' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 528) // token='break' && (break_stmt_var = break_stmt_rule(p)) // break_stmt ) @@ -1826,7 +1828,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'continue' continue_stmt")); stmt_ty continue_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 528) // token='continue' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 529) // token='continue' && (continue_stmt_var = continue_stmt_rule(p)) // continue_stmt ) @@ -1847,7 +1849,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'global' global_stmt")); stmt_ty global_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 529) // token='global' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 530) // token='global' && (global_stmt_var = global_stmt_rule(p)) // global_stmt ) @@ -1868,7 +1870,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'nonlocal' nonlocal_stmt")); stmt_ty nonlocal_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 530) // token='nonlocal' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 531) // token='nonlocal' && (nonlocal_stmt_var = nonlocal_stmt_rule(p)) // nonlocal_stmt ) @@ -1938,7 +1940,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'if' if_stmt")); stmt_ty if_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 686) // token='if' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 687) // token='if' && (if_stmt_var = if_stmt_rule(p)) // if_stmt ) @@ -2022,7 +2024,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'try' try_stmt")); stmt_ty try_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 660) // token='try' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 661) // token='try' && (try_stmt_var = try_stmt_rule(p)) // try_stmt ) @@ -2043,7 +2045,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'while' while_stmt")); stmt_ty while_stmt_var; if ( - _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 693) // token='while' + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 694) // token='while' && (while_stmt_var = while_stmt_rule(p)) // while_stmt ) @@ -2769,7 +2771,11 @@ return_stmt_rule(Parser *p) return _res; } -// raise_stmt: invalid_raise_stmt | 'raise' expression ['from' expression] | 'raise' +// raise_stmt: +// | 'raise' expression 'from' expression +// | invalid_raise_stmt +// | 'raise' expression +// | 'raise' static stmt_ty raise_stmt_rule(Parser *p) { @@ -2791,6 +2797,48 @@ raise_stmt_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro + { // 'raise' expression 'from' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from' expression")); + Token * _keyword; + Token * _keyword_1; + expr_ty a; + expr_ty b; + if ( + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' + && + (a = expression_rule(p)) // expression + && + (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='from' + && + (b = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from' expression")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_Raise ( a , b , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression 'from' expression")); + } if (p->call_invalid_rules) { // invalid_raise_stmt if (p->error_indicator) { p->level--; @@ -2810,24 +2858,21 @@ raise_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_raise_stmt")); } - { // 'raise' expression ['from' expression] + { // 'raise' expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); + D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise' expression")); Token * _keyword; expr_ty a; - void *b; if ( - (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' && (a = expression_rule(p)) // expression - && - (b = _tmp_13_rule(p), !p->error_indicator) // ['from' expression] ) { - D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression ['from' expression]")); + D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -2837,7 +2882,7 @@ raise_stmt_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_Raise ( a , b , EXTRA ); + _res = _PyAST_Raise ( a , NULL , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -2847,7 +2892,7 @@ raise_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s raise_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression ['from' expression]")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'raise' expression")); } { // 'raise' if (p->error_indicator) { @@ -2857,7 +2902,7 @@ raise_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> raise_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'raise'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' ) { D(fprintf(stderr, "%*c+ raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise'")); @@ -2918,7 +2963,7 @@ pass_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> pass_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'pass'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 526)) // token='pass' + (_keyword = _PyPegen_expect_token(p, 527)) // token='pass' ) { D(fprintf(stderr, "%*c+ pass_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'pass'")); @@ -2979,7 +3024,7 @@ break_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> break_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'break'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 527)) // token='break' + (_keyword = _PyPegen_expect_token(p, 528)) // token='break' ) { D(fprintf(stderr, "%*c+ break_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'break'")); @@ -3040,7 +3085,7 @@ continue_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> continue_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'continue'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 528)) // token='continue' + (_keyword = _PyPegen_expect_token(p, 529)) // token='continue' ) { D(fprintf(stderr, "%*c+ continue_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'continue'")); @@ -3102,9 +3147,9 @@ global_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 529)) // token='global' + (_keyword = _PyPegen_expect_token(p, 530)) // token='global' && - (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_14_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ global_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'global' ','.NAME+")); @@ -3166,9 +3211,9 @@ nonlocal_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 530)) // token='nonlocal' + (_keyword = _PyPegen_expect_token(p, 531)) // token='nonlocal' && - (a = (asdl_expr_seq*)_gather_15_rule(p)) // ','.NAME+ + (a = (asdl_expr_seq*)_gather_14_rule(p)) // ','.NAME+ ) { D(fprintf(stderr, "%*c+ nonlocal_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'nonlocal' ','.NAME+")); @@ -3230,11 +3275,11 @@ del_stmt_rule(Parser *p) Token * _keyword; asdl_expr_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='del' + (_keyword = _PyPegen_expect_token(p, 630)) // token='del' && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, _tmp_16_rule, p) + _PyPegen_lookahead(1, _tmp_15_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -3377,11 +3422,11 @@ assert_stmt_rule(Parser *p) expr_ty a; void *b; if ( - (_keyword = _PyPegen_expect_token(p, 532)) // token='assert' + (_keyword = _PyPegen_expect_token(p, 533)) // token='assert' && (a = expression_rule(p)) // expression && - (b = _tmp_17_rule(p), !p->error_indicator) // [',' expression] + (b = _tmp_16_rule(p), !p->error_indicator) // [',' expression] ) { D(fprintf(stderr, "%*c+ assert_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'assert' expression [',' expression]")); @@ -3519,7 +3564,7 @@ import_name_rule(Parser *p) Token * _keyword; asdl_alias_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='import' + (_keyword = _PyPegen_expect_token(p, 639)) // token='import' && (a = dotted_as_names_rule(p)) // dotted_as_names ) @@ -3588,13 +3633,13 @@ import_from_rule(Parser *p) expr_ty b; asdl_alias_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='from' + (_keyword = _PyPegen_expect_token(p, 638)) // token='from' && - (a = _loop0_18_rule(p)) // (('.' | '...'))* + (a = _loop0_17_rule(p)) // (('.' | '...'))* && (b = dotted_name_rule(p)) // dotted_name && - (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 639)) // token='import' && (c = import_from_targets_rule(p)) // import_from_targets ) @@ -3632,11 +3677,11 @@ import_from_rule(Parser *p) asdl_seq * a; asdl_alias_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='from' + (_keyword = _PyPegen_expect_token(p, 638)) // token='from' && - (a = _loop1_19_rule(p)) // (('.' | '...'))+ + (a = _loop1_18_rule(p)) // (('.' | '...'))+ && - (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='import' + (_keyword_1 = _PyPegen_expect_token(p, 639)) // token='import' && (b = import_from_targets_rule(p)) // import_from_targets ) @@ -3829,7 +3874,7 @@ import_from_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> import_from_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_21_rule(p)) // ','.import_from_as_name+ + (a = (asdl_alias_seq*)_gather_20_rule(p)) // ','.import_from_as_name+ ) { D(fprintf(stderr, "%*c+ import_from_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); @@ -3903,7 +3948,7 @@ import_from_as_name_rule(Parser *p) if ( (a = _PyPegen_name_token(p)) // NAME && - (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME ['as' NAME]")); @@ -3955,7 +4000,7 @@ dotted_as_names_rule(Parser *p) D(fprintf(stderr, "%*c> dotted_as_names[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); asdl_alias_seq* a; if ( - (a = (asdl_alias_seq*)_gather_24_rule(p)) // ','.dotted_as_name+ + (a = (asdl_alias_seq*)_gather_23_rule(p)) // ','.dotted_as_name+ ) { D(fprintf(stderr, "%*c+ dotted_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); @@ -4029,7 +4074,7 @@ dotted_as_name_rule(Parser *p) if ( (a = dotted_name_rule(p)) // dotted_name && - (b = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (b = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name ['as' NAME]")); @@ -4280,7 +4325,7 @@ decorators_rule(Parser *p) D(fprintf(stderr, "%*c> decorators[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_25_rule(p)) // (('@' named_expression NEWLINE))+ + (a = (asdl_expr_seq*)_loop1_24_rule(p)) // (('@' named_expression NEWLINE))+ ) { D(fprintf(stderr, "%*c+ decorators[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); @@ -4423,13 +4468,13 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 705)) // token='class' + (_keyword = _PyPegen_expect_token(p, 706)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (b = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (b = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4590,7 +4635,7 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='def' + (_keyword = _PyPegen_expect_token(p, 704)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4602,7 +4647,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4651,9 +4696,9 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 703)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 704)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && @@ -4665,7 +4710,7 @@ function_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (a = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (a = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -4790,9 +4835,9 @@ parameters_rule(Parser *p) if ( (a = slash_no_default_rule(p)) // slash_no_default && - (b = (asdl_arg_seq*)_loop0_28_rule(p)) // param_no_default* + (b = (asdl_arg_seq*)_loop0_27_rule(p)) // param_no_default* && - (c = _loop0_29_rule(p)) // param_with_default* + (c = _loop0_28_rule(p)) // param_with_default* && (d = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4822,7 +4867,7 @@ parameters_rule(Parser *p) if ( (a = slash_with_default_rule(p)) // slash_with_default && - (b = _loop0_29_rule(p)) // param_with_default* + (b = _loop0_28_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4850,9 +4895,9 @@ parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && - (b = _loop0_29_rule(p)) // param_with_default* + (b = _loop0_28_rule(p)) // param_with_default* && (c = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4879,7 +4924,7 @@ parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_31_rule(p)) // param_with_default+ + (a = _loop1_30_rule(p)) // param_with_default+ && (b = star_etc_rule(p), !p->error_indicator) // star_etc? ) @@ -4950,7 +4995,7 @@ slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -4979,7 +5024,7 @@ slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_30_rule(p)) // param_no_default+ + (a = (asdl_arg_seq*)_loop1_29_rule(p)) // param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5031,9 +5076,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_28_rule(p)) // param_no_default* + (a = _loop0_27_rule(p)) // param_no_default* && - (b = _loop1_31_rule(p)) // param_with_default+ + (b = _loop1_30_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5063,9 +5108,9 @@ slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_28_rule(p)) // param_no_default* + (a = _loop0_27_rule(p)) // param_no_default* && - (b = _loop1_31_rule(p)) // param_with_default+ + (b = _loop1_30_rule(p)) // param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -5143,7 +5188,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_rule(p)) // param_no_default && - (b = _loop0_32_rule(p)) // param_maybe_default* + (b = _loop0_31_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5176,7 +5221,7 @@ star_etc_rule(Parser *p) && (a = param_no_default_star_annotation_rule(p)) // param_no_default_star_annotation && - (b = _loop0_32_rule(p)) // param_maybe_default* + (b = _loop0_31_rule(p)) // param_maybe_default* && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5209,7 +5254,7 @@ star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_33_rule(p)) // param_maybe_default+ + (b = _loop1_32_rule(p)) // param_maybe_default+ && (c = kwds_rule(p), !p->error_indicator) // kwds? ) @@ -5991,7 +6036,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6036,7 +6081,7 @@ if_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (a = named_expression_rule(p)) // named_expression && @@ -6131,7 +6176,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; stmt_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 692)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6176,7 +6221,7 @@ elif_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 692)) // token='elif' && (a = named_expression_rule(p)) // named_expression && @@ -6257,7 +6302,7 @@ else_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword = _PyPegen_expect_token(p, 691)) // token='else' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -6336,7 +6381,7 @@ while_stmt_rule(Parser *p) asdl_stmt_seq* b; void *c; if ( - (_keyword = _PyPegen_expect_token(p, 693)) // token='while' + (_keyword = _PyPegen_expect_token(p, 694)) // token='while' && (a = named_expression_rule(p)) // named_expression && @@ -6436,11 +6481,11 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 700)) // token='in' && (_cut_var = 1) && @@ -6498,13 +6543,13 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='for' && (t = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 700)) // token='in' && (_cut_var = 1) && @@ -6633,11 +6678,11 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword = _PyPegen_expect_token(p, 652)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6684,9 +6729,9 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword = _PyPegen_expect_token(p, 652)) // token='with' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6733,13 +6778,13 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 652)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -6785,11 +6830,11 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword_1 = _PyPegen_expect_token(p, 652)) // token='with' && - (a = (asdl_withitem_seq*)_gather_35_rule(p)) // ','.with_item+ + (a = (asdl_withitem_seq*)_gather_34_rule(p)) // ','.with_item+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -6873,11 +6918,11 @@ with_item_rule(Parser *p) if ( (e = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_35_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); @@ -6998,7 +7043,7 @@ try_stmt_rule(Parser *p) asdl_stmt_seq* b; asdl_stmt_seq* f; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7042,13 +7087,13 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_37_rule(p)) // except_block+ + (ex = (asdl_excepthandler_seq*)_loop1_36_rule(p)) // except_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7090,13 +7135,13 @@ try_stmt_rule(Parser *p) asdl_excepthandler_seq* ex; void *f; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && (b = block_rule(p)) // block && - (ex = (asdl_excepthandler_seq*)_loop1_38_rule(p)) // except_star_block+ + (ex = (asdl_excepthandler_seq*)_loop1_37_rule(p)) // except_star_block+ && (el = else_block_rule(p), !p->error_indicator) // else_block? && @@ -7189,7 +7234,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (e = expression_rule(p)) // expression && @@ -7233,11 +7278,11 @@ except_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7279,7 +7324,7 @@ except_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (e = expressions_rule(p)) // expressions && @@ -7320,7 +7365,7 @@ except_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -7432,7 +7477,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7479,13 +7524,13 @@ except_star_block_rule(Parser *p) expr_ty e; expr_ty t; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (e = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (t = _PyPegen_name_token(p)) // NAME && @@ -7528,7 +7573,7 @@ except_star_block_rule(Parser *p) asdl_stmt_seq* b; expr_ty e; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -7628,7 +7673,7 @@ finally_block_rule(Parser *p) Token * _literal; asdl_stmt_seq* a; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 678)) // token='finally' && (_literal = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -7702,7 +7747,7 @@ match_stmt_rule(Parser *p) && (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' && - (cases = (asdl_match_case_seq*)_loop1_39_rule(p)) // case_block+ + (cases = (asdl_match_case_seq*)_loop1_38_rule(p)) // case_block+ && (dedent_var = _PyPegen_expect_token(p, DEDENT)) // token='DEDENT' ) @@ -7936,7 +7981,7 @@ guard_rule(Parser *p) Token * _keyword; expr_ty guard; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (guard = named_expression_rule(p)) // named_expression ) @@ -8131,7 +8176,7 @@ as_pattern_rule(Parser *p) if ( (pattern = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (target = pattern_capture_target_rule(p)) // pattern_capture_target ) @@ -8213,7 +8258,7 @@ or_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> or_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); asdl_pattern_seq* patterns; if ( - (patterns = (asdl_pattern_seq*)_gather_41_rule(p)) // '|'.closed_pattern+ + (patterns = (asdl_pattern_seq*)_gather_40_rule(p)) // '|'.closed_pattern+ ) { D(fprintf(stderr, "%*c+ or_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'|'.closed_pattern+")); @@ -8466,7 +8511,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_41_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8565,7 +8610,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8598,7 +8643,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8631,7 +8676,7 @@ literal_pattern_rule(Parser *p) D(fprintf(stderr, "%*c> literal_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -8700,7 +8745,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_41_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8738,7 +8783,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_42_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -8759,7 +8804,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -8792,7 +8837,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -8825,7 +8870,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -9302,7 +9347,7 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); @@ -9417,7 +9462,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_43_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -9836,7 +9881,7 @@ maybe_sequence_pattern_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * patterns; if ( - (patterns = _gather_46_rule(p)) // ','.maybe_star_pattern+ + (patterns = _gather_45_rule(p)) // ','.maybe_star_pattern+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -10244,13 +10289,13 @@ items_pattern_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> items_pattern[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - asdl_seq * _gather_48_var; + asdl_seq * _gather_47_var; if ( - (_gather_48_var = _gather_48_rule(p)) // ','.key_value_pattern+ + (_gather_47_var = _gather_47_rule(p)) // ','.key_value_pattern+ ) { D(fprintf(stderr, "%*c+ items_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.key_value_pattern+")); - _res = _gather_48_var; + _res = _gather_47_var; goto done; } p->mark = _mark; @@ -10286,7 +10331,7 @@ key_value_pattern_rule(Parser *p) void *key; pattern_ty pattern; if ( - (key = _tmp_49_rule(p)) // literal_expr | attr + (key = _tmp_48_rule(p)) // literal_expr | attr && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -10614,7 +10659,7 @@ positional_patterns_rule(Parser *p) D(fprintf(stderr, "%*c> positional_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.pattern+")); asdl_pattern_seq* args; if ( - (args = (asdl_pattern_seq*)_gather_51_rule(p)) // ','.pattern+ + (args = (asdl_pattern_seq*)_gather_50_rule(p)) // ','.pattern+ ) { D(fprintf(stderr, "%*c+ positional_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.pattern+")); @@ -10655,13 +10700,13 @@ keyword_patterns_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> keyword_patterns[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - asdl_seq * _gather_53_var; + asdl_seq * _gather_52_var; if ( - (_gather_53_var = _gather_53_rule(p)) // ','.keyword_pattern+ + (_gather_52_var = _gather_52_rule(p)) // ','.keyword_pattern+ ) { D(fprintf(stderr, "%*c+ keyword_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.keyword_pattern+")); - _res = _gather_53_var; + _res = _gather_52_var; goto done; } p->mark = _mark; @@ -10887,7 +10932,7 @@ type_param_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_type_param_seq* a; if ( - (a = (asdl_type_param_seq*)_gather_55_rule(p)) // ','.type_param+ + (a = (asdl_type_param_seq*)_gather_54_rule(p)) // ','.type_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11257,7 +11302,7 @@ expressions_rule(Parser *p) if ( (a = expression_rule(p)) // expression && - (b = _loop1_56_rule(p)) // ((',' expression))+ + (b = _loop1_55_rule(p)) // ((',' expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11428,11 +11473,11 @@ expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='else' && (c = expression_rule(p)) // expression ) @@ -11536,9 +11581,9 @@ yield_expr_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 587)) // token='yield' + (_keyword = _PyPegen_expect_token(p, 588)) // token='yield' && - (_keyword_1 = _PyPegen_expect_token(p, 637)) // token='from' + (_keyword_1 = _PyPegen_expect_token(p, 638)) // token='from' && (a = expression_rule(p)) // expression ) @@ -11574,7 +11619,7 @@ yield_expr_rule(Parser *p) Token * _keyword; void *a; if ( - (_keyword = _PyPegen_expect_token(p, 587)) // token='yield' + (_keyword = _PyPegen_expect_token(p, 588)) // token='yield' && (a = star_expressions_rule(p), !p->error_indicator) // star_expressions? ) @@ -11645,7 +11690,7 @@ star_expressions_rule(Parser *p) if ( (a = star_expression_rule(p)) // star_expression && - (b = _loop1_57_rule(p)) // ((',' star_expression))+ + (b = _loop1_56_rule(p)) // ((',' star_expression))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -11844,7 +11889,7 @@ star_named_expressions_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_59_rule(p)) // ','.star_named_expression+ + (a = (asdl_expr_seq*)_gather_58_rule(p)) // ','.star_named_expression+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -12140,7 +12185,7 @@ disjunction_rule(Parser *p) if ( (a = conjunction_rule(p)) // conjunction && - (b = _loop1_60_rule(p)) // (('or' conjunction))+ + (b = _loop1_59_rule(p)) // (('or' conjunction))+ ) { D(fprintf(stderr, "%*c+ disjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "conjunction (('or' conjunction))+")); @@ -12228,7 +12273,7 @@ conjunction_rule(Parser *p) if ( (a = inversion_rule(p)) // inversion && - (b = _loop1_61_rule(p)) // (('and' inversion))+ + (b = _loop1_60_rule(p)) // (('and' inversion))+ ) { D(fprintf(stderr, "%*c+ conjunction[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "inversion (('and' inversion))+")); @@ -12314,7 +12359,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 707)) // token='not' + (_keyword = _PyPegen_expect_token(p, 708)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12400,7 +12445,7 @@ comparison_rule(Parser *p) if ( (a = bitwise_or_rule(p)) // bitwise_or && - (b = _loop1_62_rule(p)) // compare_op_bitwise_or_pair+ + (b = _loop1_61_rule(p)) // compare_op_bitwise_or_pair+ ) { D(fprintf(stderr, "%*c+ comparison[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or compare_op_bitwise_or_pair+")); @@ -12734,10 +12779,10 @@ noteq_bitwise_or_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> noteq_bitwise_or[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('!=') bitwise_or")); - void *_tmp_63_var; + void *_tmp_62_var; expr_ty a; if ( - (_tmp_63_var = _tmp_63_rule(p)) // '!=' + (_tmp_62_var = _tmp_62_rule(p)) // '!=' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -12968,9 +13013,9 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 707)) // token='not' + (_keyword = _PyPegen_expect_token(p, 708)) // token='not' && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 700)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13016,7 +13061,7 @@ in_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword = _PyPegen_expect_token(p, 700)) // token='in' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13063,9 +13108,9 @@ isnot_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 596)) // token='is' + (_keyword = _PyPegen_expect_token(p, 597)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 707)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 708)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -13111,7 +13156,7 @@ is_bitwise_or_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 596)) // token='is' + (_keyword = _PyPegen_expect_token(p, 597)) // token='is' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -14427,7 +14472,7 @@ await_primary_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 597)) // token='await' + (_keyword = _PyPegen_expect_token(p, 598)) // token='await' && (a = primary_rule(p)) // primary ) @@ -14785,7 +14830,7 @@ slices_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_65_rule(p)) // ','.(slice | starred_expression)+ + (a = (asdl_expr_seq*)_gather_64_rule(p)) // ','.(slice | starred_expression)+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -14857,7 +14902,7 @@ slice_rule(Parser *p) && (b = expression_rule(p), !p->error_indicator) // expression? && - (c = _tmp_66_rule(p), !p->error_indicator) // [':' expression?] + (c = _tmp_65_rule(p), !p->error_indicator) // [':' expression?] ) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression? ':' expression? [':' expression?]")); @@ -14971,7 +15016,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -15004,7 +15049,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -15037,7 +15082,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -15070,7 +15115,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_42_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -15108,15 +15153,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - void *_tmp_67_var; + void *_tmp_66_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7) // token='(' && - (_tmp_67_var = _tmp_67_rule(p)) // tuple | group | genexp + (_tmp_66_var = _tmp_66_rule(p)) // tuple | group | genexp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'(' (tuple | group | genexp)")); - _res = _tmp_67_var; + _res = _tmp_66_var; goto done; } p->mark = _mark; @@ -15129,15 +15174,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - void *_tmp_68_var; + void *_tmp_67_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9) // token='[' && - (_tmp_68_var = _tmp_68_rule(p)) // list | listcomp + (_tmp_67_var = _tmp_67_rule(p)) // list | listcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'[' (list | listcomp)")); - _res = _tmp_68_var; + _res = _tmp_67_var; goto done; } p->mark = _mark; @@ -15150,15 +15195,15 @@ atom_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - void *_tmp_69_var; + void *_tmp_68_var; if ( _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25) // token='{' && - (_tmp_69_var = _tmp_69_rule(p)) // dict | set | dictcomp | setcomp + (_tmp_68_var = _tmp_68_rule(p)) // dict | set | dictcomp | setcomp ) { D(fprintf(stderr, "%*c+ atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "&'{' (dict | set | dictcomp | setcomp)")); - _res = _tmp_69_var; + _res = _tmp_68_var; goto done; } p->mark = _mark; @@ -15229,7 +15274,7 @@ group_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_70_rule(p)) // yield_expr | named_expression + (a = _tmp_69_rule(p)) // yield_expr | named_expression && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -15305,7 +15350,7 @@ lambdef_rule(Parser *p) void *a; expr_ty b; if ( - (_keyword = _PyPegen_expect_token(p, 621)) // token='lambda' + (_keyword = _PyPegen_expect_token(p, 622)) // token='lambda' && (a = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -15430,9 +15475,9 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_no_default_rule(p)) // lambda_slash_no_default && - (b = (asdl_arg_seq*)_loop0_71_rule(p)) // lambda_param_no_default* + (b = (asdl_arg_seq*)_loop0_70_rule(p)) // lambda_param_no_default* && - (c = _loop0_72_rule(p)) // lambda_param_with_default* + (c = _loop0_71_rule(p)) // lambda_param_with_default* && (d = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15462,7 +15507,7 @@ lambda_parameters_rule(Parser *p) if ( (a = lambda_slash_with_default_rule(p)) // lambda_slash_with_default && - (b = _loop0_72_rule(p)) // lambda_param_with_default* + (b = _loop0_71_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15490,9 +15535,9 @@ lambda_parameters_rule(Parser *p) asdl_seq * b; void *c; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && - (b = _loop0_72_rule(p)) // lambda_param_with_default* + (b = _loop0_71_rule(p)) // lambda_param_with_default* && (c = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15519,7 +15564,7 @@ lambda_parameters_rule(Parser *p) asdl_seq * a; void *b; if ( - (a = _loop1_74_rule(p)) // lambda_param_with_default+ + (a = _loop1_73_rule(p)) // lambda_param_with_default+ && (b = lambda_star_etc_rule(p), !p->error_indicator) // lambda_star_etc? ) @@ -15592,7 +15637,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal_1; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15621,7 +15666,7 @@ lambda_slash_no_default_rule(Parser *p) Token * _literal; asdl_arg_seq* a; if ( - (a = (asdl_arg_seq*)_loop1_73_rule(p)) // lambda_param_no_default+ + (a = (asdl_arg_seq*)_loop1_72_rule(p)) // lambda_param_no_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15673,9 +15718,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_71_rule(p)) // lambda_param_no_default* + (a = _loop0_70_rule(p)) // lambda_param_no_default* && - (b = _loop1_74_rule(p)) // lambda_param_with_default+ + (b = _loop1_73_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15705,9 +15750,9 @@ lambda_slash_with_default_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _loop0_71_rule(p)) // lambda_param_no_default* + (a = _loop0_70_rule(p)) // lambda_param_no_default* && - (b = _loop1_74_rule(p)) // lambda_param_with_default+ + (b = _loop1_73_rule(p)) // lambda_param_with_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -15784,7 +15829,7 @@ lambda_star_etc_rule(Parser *p) && (a = lambda_param_no_default_rule(p)) // lambda_param_no_default && - (b = _loop0_75_rule(p)) // lambda_param_maybe_default* + (b = _loop0_74_rule(p)) // lambda_param_maybe_default* && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -15817,7 +15862,7 @@ lambda_star_etc_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _loop1_76_rule(p)) // lambda_param_maybe_default+ + (b = _loop1_75_rule(p)) // lambda_param_maybe_default+ && (c = lambda_kwds_rule(p), !p->error_indicator) // lambda_kwds? ) @@ -16457,7 +16502,7 @@ fstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_77_rule(p)) // fstring_format_spec* + (spec = _loop0_76_rule(p)) // fstring_format_spec* ) { D(fprintf(stderr, "%*c+ fstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' fstring_format_spec*")); @@ -16575,7 +16620,7 @@ fstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' && - (b = _loop0_78_rule(p)) // fstring_middle* + (b = _loop0_77_rule(p)) // fstring_middle* && (c = _PyPegen_expect_token(p, FSTRING_END)) // token='FSTRING_END' ) @@ -16791,7 +16836,7 @@ tstring_full_format_spec_rule(Parser *p) if ( (colon = _PyPegen_expect_token(p, 11)) // token=':' && - (spec = _loop0_79_rule(p)) // tstring_format_spec* + (spec = _loop0_78_rule(p)) // tstring_format_spec* ) { D(fprintf(stderr, "%*c+ tstring_full_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' tstring_format_spec*")); @@ -17010,7 +17055,7 @@ tstring_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, TSTRING_START)) // token='TSTRING_START' && - (b = _loop0_80_rule(p)) // tstring_middle* + (b = _loop0_79_rule(p)) // tstring_middle* && (c = _PyPegen_expect_token(p, TSTRING_END)) // token='TSTRING_END' ) @@ -17078,7 +17123,7 @@ string_rule(Parser *p) return _res; } -// strings: ((fstring | string | tstring))+ +// strings: invalid_string_tstring_concat | ((fstring | string))+ | tstring+ static expr_ty strings_rule(Parser *p) { @@ -17104,18 +17149,37 @@ strings_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // ((fstring | string | tstring))+ + if (p->call_invalid_rules) { // invalid_string_tstring_concat + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + void *invalid_string_tstring_concat_var; + if ( + (invalid_string_tstring_concat_var = invalid_string_tstring_concat_rule(p)) // invalid_string_tstring_concat + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + _res = invalid_string_tstring_concat_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_string_tstring_concat")); + } + { // ((fstring | string))+ if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string | tstring))+ + (a = (asdl_expr_seq*)_loop1_80_rule(p)) // ((fstring | string))+ ) { - D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -17135,7 +17199,40 @@ strings_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string | tstring))+")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+")); + } + { // tstring+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+")); + asdl_expr_seq* a; + if ( + (a = (asdl_expr_seq*)_loop1_81_rule(p)) // tstring+ + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyPegen_concatenate_tstrings ( p , a , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+")); } _res = NULL; done: @@ -17672,13 +17769,13 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_2 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_2 = _PyPegen_expect_token(p, 700)) // token='in' && (_cut_var = 1) && @@ -17717,11 +17814,11 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' && (a = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 700)) // token='in' && (_cut_var = 1) && @@ -21022,11 +21119,11 @@ expression_without_invalid_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='else' && (c = expression_rule(p)) // expression ) @@ -21326,7 +21423,7 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (b = disjunction_rule(p)) // disjunction && @@ -21359,11 +21456,11 @@ invalid_expression_rule(Parser *p) if ( (a = disjunction_rule(p)) // disjunction && - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='else' && _PyPegen_lookahead_for_expr(0, expression_rule, p) ) @@ -21395,11 +21492,11 @@ invalid_expression_rule(Parser *p) if ( (a = (stmt_ty)_tmp_116_rule(p)) // pass_stmt | break_stmt | continue_stmt && - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (b = disjunction_rule(p)) // disjunction && - (_keyword_1 = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='else' && (c = simple_stmt_rule(p)) // simple_stmt ) @@ -21428,7 +21525,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 621)) // token='lambda' + (a = _PyPegen_expect_token(p, 622)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -21461,7 +21558,7 @@ invalid_expression_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 621)) // token='lambda' + (a = _PyPegen_expect_token(p, 622)) // token='lambda' && (_opt_var = lambda_params_rule(p), !p->error_indicator) // lambda_params? && @@ -21932,9 +22029,9 @@ invalid_raise_stmt_rule(Parser *p) Token * a; Token * b; if ( - (a = _PyPegen_expect_token(p, 627)) // token='raise' + (a = _PyPegen_expect_token(p, 628)) // token='raise' && - (b = _PyPegen_expect_token(p, 637)) // token='from' + (b = _PyPegen_expect_token(p, 638)) // token='from' ) { D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' 'from'")); @@ -21960,11 +22057,11 @@ invalid_raise_stmt_rule(Parser *p) Token * a; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 627)) // token='raise' + (_keyword = _PyPegen_expect_token(p, 628)) // token='raise' && (expression_var = expression_rule(p)) // expression && - (a = _PyPegen_expect_token(p, 637)) // token='from' + (a = _PyPegen_expect_token(p, 638)) // token='from' ) { D(fprintf(stderr, "%*c+ invalid_raise_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'raise' expression 'from'")); @@ -22008,7 +22105,7 @@ invalid_del_stmt_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 629)) // token='del' + (_keyword = _PyPegen_expect_token(p, 630)) // token='del' && (a = star_expressions_rule(p)) // star_expressions ) @@ -22305,13 +22402,13 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); - asdl_seq * _loop0_32_var; + asdl_seq * _loop0_31_var; void *_tmp_123_var; Token * a; if ( (_tmp_123_var = _tmp_123_rule(p)) // slash_no_default | slash_with_default && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22335,7 +22432,7 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); - asdl_seq * _loop0_28_var; + asdl_seq * _loop0_27_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22343,7 +22440,7 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = slash_no_default_rule(p), !p->error_indicator) // slash_no_default? && - (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* + (_loop0_27_var = _loop0_27_rule(p)) // param_no_default* && (invalid_parameters_helper_var = invalid_parameters_helper_rule(p)) // invalid_parameters_helper && @@ -22369,18 +22466,18 @@ invalid_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); - asdl_seq * _loop0_28_var; - asdl_seq * _loop1_30_var; + asdl_seq * _loop0_27_var; + asdl_seq * _loop1_29_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_28_var = _loop0_28_rule(p)) // param_no_default* + (_loop0_27_var = _loop0_27_rule(p)) // param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_loop1_30_var = _loop1_30_rule(p)) // param_no_default+ + (_loop1_29_var = _loop1_29_rule(p)) // param_no_default+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22407,8 +22504,8 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_32_var; - asdl_seq * _loop0_32_var_1; + asdl_seq * _loop0_31_var; + asdl_seq * _loop0_31_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_tmp_124_var; @@ -22416,13 +22513,13 @@ invalid_parameters_rule(Parser *p) if ( (_opt_var = _tmp_123_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (_tmp_124_var = _tmp_124_rule(p)) // ',' | param_no_default && - (_loop0_32_var_1 = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var_1 = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22447,10 +22544,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_33_var; + asdl_seq * _loop1_32_var; Token * a; if ( - (_loop1_33_var = _loop1_33_rule(p)) // param_maybe_default+ + (_loop1_32_var = _loop1_32_rule(p)) // param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -22632,7 +22729,7 @@ invalid_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_32_var; + asdl_seq * _loop0_31_var; void *_tmp_127_var; void *_tmp_127_var_1; Token * a; @@ -22641,7 +22738,7 @@ invalid_star_etc_rule(Parser *p) && (_tmp_127_var = _tmp_127_rule(p)) // param_no_default | ',' && - (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* + (_loop0_31_var = _loop0_31_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && @@ -22825,13 +22922,13 @@ invalid_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - asdl_seq * _loop1_31_var; + asdl_seq * _loop1_30_var; if ( - (_loop1_31_var = _loop1_31_rule(p)) // param_with_default+ + (_loop1_30_var = _loop1_30_rule(p)) // param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+")); - _res = _loop1_31_var; + _res = _loop1_30_var; goto done; } p->mark = _mark; @@ -22896,13 +22993,13 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); - asdl_seq * _loop0_75_var; + asdl_seq * _loop0_74_var; void *_tmp_129_var; Token * a; if ( (_tmp_129_var = _tmp_129_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -22926,7 +23023,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); - asdl_seq * _loop0_71_var; + asdl_seq * _loop0_70_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings arg_ty a; @@ -22934,7 +23031,7 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = lambda_slash_no_default_rule(p), !p->error_indicator) // lambda_slash_no_default? && - (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* + (_loop0_70_var = _loop0_70_rule(p)) // lambda_param_no_default* && (invalid_lambda_parameters_helper_var = invalid_lambda_parameters_helper_rule(p)) // invalid_lambda_parameters_helper && @@ -22961,13 +23058,13 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); asdl_seq * _gather_131_var; - asdl_seq * _loop0_71_var; + asdl_seq * _loop0_70_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; if ( - (_loop0_71_var = _loop0_71_rule(p)) // lambda_param_no_default* + (_loop0_70_var = _loop0_70_rule(p)) // lambda_param_no_default* && (a = _PyPegen_expect_token(p, 7)) // token='(' && @@ -22998,8 +23095,8 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); Token * _literal; - asdl_seq * _loop0_75_var; - asdl_seq * _loop0_75_var_1; + asdl_seq * _loop0_74_var; + asdl_seq * _loop0_74_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_tmp_132_var; @@ -23007,13 +23104,13 @@ invalid_lambda_parameters_rule(Parser *p) if ( (_opt_var = _tmp_129_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (_tmp_132_var = _tmp_132_rule(p)) // ',' | lambda_param_no_default && - (_loop0_75_var_1 = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var_1 = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 17)) // token='/' ) @@ -23038,10 +23135,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); Token * _literal; - asdl_seq * _loop1_76_var; + asdl_seq * _loop1_75_var; Token * a; if ( - (_loop1_76_var = _loop1_76_rule(p)) // lambda_param_maybe_default+ + (_loop1_75_var = _loop1_75_rule(p)) // lambda_param_maybe_default+ && (_literal = _PyPegen_expect_token(p, 17)) // token='/' && @@ -23112,13 +23209,13 @@ invalid_lambda_parameters_helper_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters_helper[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - asdl_seq * _loop1_74_var; + asdl_seq * _loop1_73_var; if ( - (_loop1_74_var = _loop1_74_rule(p)) // lambda_param_with_default+ + (_loop1_73_var = _loop1_73_rule(p)) // lambda_param_with_default+ ) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+")); - _res = _loop1_74_var; + _res = _loop1_73_var; goto done; } p->mark = _mark; @@ -23211,7 +23308,7 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; - asdl_seq * _loop0_75_var; + asdl_seq * _loop0_74_var; void *_tmp_134_var; void *_tmp_134_var_1; Token * a; @@ -23220,7 +23317,7 @@ invalid_lambda_star_etc_rule(Parser *p) && (_tmp_134_var = _tmp_134_rule(p)) // lambda_param_no_default | ',' && - (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* + (_loop0_74_var = _loop0_74_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && @@ -23444,11 +23541,11 @@ invalid_with_item_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_35_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); @@ -23494,13 +23591,13 @@ invalid_for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_135_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' && (_tmp_135_var = _tmp_135_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && - _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 699) // token='in' + _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 700) // token='in' ) { D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); @@ -23546,9 +23643,9 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' && (a = star_expressions_rule(p)) // star_expressions ) @@ -23678,11 +23775,11 @@ invalid_import_rule(Parser *p) Token * a; expr_ty dotted_name_var; if ( - (a = _PyPegen_expect_token(p, 638)) // token='import' + (a = _PyPegen_expect_token(p, 639)) // token='import' && (_gather_137_var = _gather_137_rule(p)) // ','.dotted_name+ && - (_keyword = _PyPegen_expect_token(p, 637)) // token='from' + (_keyword = _PyPegen_expect_token(p, 638)) // token='from' && (dotted_name_var = dotted_name_rule(p)) // dotted_name ) @@ -23709,7 +23806,7 @@ invalid_import_rule(Parser *p) Token * _keyword; Token * token; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='import' + (_keyword = _PyPegen_expect_token(p, 639)) // token='import' && (token = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23758,7 +23855,7 @@ invalid_dotted_as_name_rule(Parser *p) if ( (dotted_name_var = dotted_name_rule(p)) // dotted_name && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && _PyPegen_lookahead(0, _tmp_138_rule, p) && @@ -23809,7 +23906,7 @@ invalid_import_from_as_name_rule(Parser *p) if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && _PyPegen_lookahead(0, _tmp_138_rule, p) && @@ -23935,9 +24032,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword = _PyPegen_expect_token(p, 652)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -23973,9 +24070,9 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword = _PyPegen_expect_token(p, 652)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -24035,9 +24132,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 651)) // token='with' + (a = _PyPegen_expect_token(p, 652)) // token='with' && (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ && @@ -24078,9 +24175,9 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 651)) // token='with' + (a = _PyPegen_expect_token(p, 652)) // token='with' && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && @@ -24143,7 +24240,7 @@ invalid_try_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 660)) // token='try' + (a = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24175,7 +24272,7 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24207,28 +24304,28 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; Token * _literal_1; asdl_seq * _loop0_144_var; - asdl_seq * _loop1_37_var; + asdl_seq * _loop1_36_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; Token * b; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (_loop0_144_var = _loop0_144_rule(p)) // block* && - (_loop1_37_var = _loop1_37_rule(p)) // except_block+ + (_loop1_36_var = _loop1_36_rule(p)) // except_block+ && - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (b = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -24256,20 +24353,20 @@ invalid_try_stmt_rule(Parser *p) Token * _literal; Token * _literal_1; asdl_seq * _loop0_144_var; - asdl_seq * _loop1_38_var; + asdl_seq * _loop1_37_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings Token * a; if ( - (_keyword = _PyPegen_expect_token(p, 660)) // token='try' + (_keyword = _PyPegen_expect_token(p, 661)) // token='try' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (_loop0_144_var = _loop0_144_rule(p)) // block* && - (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ + (_loop1_37_var = _loop1_37_rule(p)) // except_star_block+ && - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (_opt_var = _tmp_145_rule(p), !p->error_indicator) // [expression ['as' NAME]] && @@ -24326,7 +24423,7 @@ invalid_except_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (a = expression_rule(p)) // expression && @@ -24334,7 +24431,7 @@ invalid_except_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24366,11 +24463,11 @@ invalid_except_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24397,7 +24494,7 @@ invalid_except_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24428,11 +24525,11 @@ invalid_except_stmt_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (a = expression_rule(p)) // expression && @@ -24492,7 +24589,7 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expressions_var; expr_ty name_var; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24502,7 +24599,7 @@ invalid_except_star_stmt_rule(Parser *p) && (expressions_var = expressions_rule(p)) // expressions && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24535,13 +24632,13 @@ invalid_except_star_stmt_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24569,7 +24666,7 @@ invalid_except_star_stmt_rule(Parser *p) void *_tmp_146_var; Token * a; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && @@ -24603,13 +24700,13 @@ invalid_except_star_stmt_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty expression_var; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_keyword_1 = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword_1 = _PyPegen_expect_token(p, 685)) // token='as' && (a = expression_rule(p)) // expression && @@ -24660,7 +24757,7 @@ invalid_finally_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='finally' + (a = _PyPegen_expect_token(p, 678)) // token='finally' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24716,11 +24813,11 @@ invalid_except_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24752,7 +24849,7 @@ invalid_except_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24808,13 +24905,13 @@ invalid_except_star_stmt_indent_rule(Parser *p) expr_ty expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 681)) // token='except' + (a = _PyPegen_expect_token(p, 682)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25047,7 +25144,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (a = _PyPegen_expect_soft_keyword(p, "_")) // soft_keyword='"_"' ) @@ -25077,7 +25174,7 @@ invalid_as_pattern_rule(Parser *p) if ( (or_pattern_var = or_pattern_rule(p)) // or_pattern && - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (a = expression_rule(p)) // expression ) @@ -25229,7 +25326,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25260,7 +25357,7 @@ invalid_if_stmt_rule(Parser *p) expr_ty a_1; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 686)) // token='if' + (a = _PyPegen_expect_token(p, 687)) // token='if' && (a_1 = named_expression_rule(p)) // named_expression && @@ -25315,7 +25412,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 691)) // token='elif' + (_keyword = _PyPegen_expect_token(p, 692)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25346,7 +25443,7 @@ invalid_elif_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 691)) // token='elif' + (a = _PyPegen_expect_token(p, 692)) // token='elif' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25399,7 +25496,7 @@ invalid_else_stmt_rule(Parser *p) Token * a; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 690)) // token='else' + (a = _PyPegen_expect_token(p, 691)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25432,13 +25529,13 @@ invalid_else_stmt_rule(Parser *p) Token * _literal; asdl_stmt_seq* block_var; if ( - (_keyword = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword = _PyPegen_expect_token(p, 691)) // token='else' && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && (block_var = block_rule(p)) // block && - (_keyword_1 = _PyPegen_expect_token(p, 691)) // token='elif' + (_keyword_1 = _PyPegen_expect_token(p, 692)) // token='elif' ) { D(fprintf(stderr, "%*c+ invalid_else_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':' block 'elif'")); @@ -25485,7 +25582,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 693)) // token='while' + (_keyword = _PyPegen_expect_token(p, 694)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25516,7 +25613,7 @@ invalid_while_stmt_rule(Parser *p) expr_ty named_expression_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 693)) // token='while' + (a = _PyPegen_expect_token(p, 694)) // token='while' && (named_expression_var = named_expression_rule(p)) // named_expression && @@ -25575,13 +25672,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword_1 = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword_1 = _PyPegen_expect_token(p, 700)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25616,13 +25713,13 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 698)) // token='for' + (a = _PyPegen_expect_token(p, 699)) // token='for' && (star_targets_var = star_targets_rule(p)) // star_targets && - (_keyword = _PyPegen_expect_token(p, 699)) // token='in' + (_keyword = _PyPegen_expect_token(p, 700)) // token='in' && (star_expressions_var = star_expressions_rule(p)) // star_expressions && @@ -25688,9 +25785,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 703)) // token='def' + (a = _PyPegen_expect_token(p, 704)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25702,7 +25799,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -25747,9 +25844,9 @@ invalid_def_raw_rule(Parser *p) asdl_stmt_seq* block_var; expr_ty name_var; if ( - (_opt_var = _PyPegen_expect_token(p, 702), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 703), !p->error_indicator) // 'async'? && - (_keyword = _PyPegen_expect_token(p, 703)) // token='def' + (_keyword = _PyPegen_expect_token(p, 704)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -25761,7 +25858,7 @@ invalid_def_raw_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - (_opt_var_3 = _tmp_27_rule(p), !p->error_indicator) // ['->' expression] + (_opt_var_3 = _tmp_26_rule(p), !p->error_indicator) // ['->' expression] && (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' && @@ -25813,13 +25910,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 705)) // token='class' + (_keyword = _PyPegen_expect_token(p, 706)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -25852,13 +25949,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 705)) // token='class' + (a = _PyPegen_expect_token(p, 706)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_26_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_25_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26493,7 +26590,7 @@ invalid_fstring_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_fstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_77_var; + asdl_seq * _loop0_76_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -26510,7 +26607,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_77_var = _loop0_77_rule(p)) // fstring_format_spec* + (_loop0_76_var = _loop0_76_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26935,7 +27032,7 @@ invalid_tstring_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_tstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_77_var; + asdl_seq * _loop0_76_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -26952,7 +27049,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_77_var = _loop0_77_rule(p)) // fstring_format_spec* + (_loop0_76_var = _loop0_76_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -27084,6 +27181,81 @@ invalid_tstring_conversion_character_rule(Parser *p) return _res; } +// invalid_string_tstring_concat: +// | ((fstring | string))+ tstring +// | tstring+ (fstring | string) +static void * +invalid_string_tstring_concat_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ((fstring | string))+ tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_80_rule(p)) // ((fstring | string))+ + && + (b = (expr_ty)tstring_rule(p)) // tstring + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+ tstring")); + } + { // tstring+ (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_81_rule(p)) // tstring+ + && + (b = (expr_ty)_tmp_153_rule(p)) // fstring | string + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+ (fstring | string)")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_arithmetic: sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion static void * invalid_arithmetic_rule(Parser *p) @@ -27103,16 +27275,16 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_153_var; + void *_tmp_154_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_153_var = _tmp_153_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 707)) // token='not' + (a = _PyPegen_expect_token(p, 708)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -27155,13 +27327,13 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_154_var; + void *_tmp_155_var; Token * a; expr_ty b; if ( - (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '~' + (_tmp_155_var = _tmp_155_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 707)) // token='not' + (a = _PyPegen_expect_token(p, 708)) // token='not' && (b = factor_rule(p)) // factor ) @@ -27508,7 +27680,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 638)) // token='import' + (_keyword = _PyPegen_expect_token(p, 639)) // token='import' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import'")); @@ -27527,7 +27699,7 @@ _tmp_5_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_5[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='from' + (_keyword = _PyPegen_expect_token(p, 638)) // token='from' ) { D(fprintf(stderr, "%*c+ _tmp_5[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from'")); @@ -27565,7 +27737,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 703)) // token='def' + (_keyword = _PyPegen_expect_token(p, 704)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -27603,7 +27775,7 @@ _tmp_6_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_6[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_6[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27641,7 +27813,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 705)) // token='class' + (_keyword = _PyPegen_expect_token(p, 706)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -27698,7 +27870,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'with'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 651)) // token='with' + (_keyword = _PyPegen_expect_token(p, 652)) // token='with' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'with'")); @@ -27717,7 +27889,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27755,7 +27927,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'for'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 698)) // token='for' + (_keyword = _PyPegen_expect_token(p, 699)) // token='for' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for'")); @@ -27774,7 +27946,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 702)) // token='async' + (_keyword = _PyPegen_expect_token(p, 703)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -27933,12 +28105,12 @@ _loop1_12_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_156_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_156_var = _tmp_156_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_156_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27977,55 +28149,9 @@ _loop1_12_rule(Parser *p) return _seq; } -// _tmp_13: 'from' expression -static void * -_tmp_13_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // 'from' expression - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'from' expression")); - Token * _keyword; - expr_ty z; - if ( - (_keyword = _PyPegen_expect_token(p, 637)) // token='from' - && - (z = expression_rule(p)) // expression - ) - { - D(fprintf(stderr, "%*c+ _tmp_13[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' expression")); - _res = z; - if (_res == NULL && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_13[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'from' expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _loop0_14: ',' NAME +// _loop0_13: ',' NAME static asdl_seq * -_loop0_14_rule(Parser *p) +_loop0_13_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28050,7 +28176,7 @@ _loop0_14_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); + D(fprintf(stderr, "%*c> _loop0_13[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' NAME")); Token * _literal; expr_ty elem; while ( @@ -28082,7 +28208,7 @@ _loop0_14_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_14[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_13[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' NAME")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28099,9 +28225,9 @@ _loop0_14_rule(Parser *p) return _seq; } -// _gather_15: NAME _loop0_14 +// _gather_14: NAME _loop0_13 static asdl_seq * -_gather_15_rule(Parser *p) +_gather_14_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28112,27 +28238,27 @@ _gather_15_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // NAME _loop0_14 + { // NAME _loop0_13 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c> _gather_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME _loop0_13")); expr_ty elem; asdl_seq * seq; if ( (elem = _PyPegen_name_token(p)) // NAME && - (seq = _loop0_14_rule(p)) // _loop0_14 + (seq = _loop0_13_rule(p)) // _loop0_13 ) { - D(fprintf(stderr, "%*c+ _gather_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c+ _gather_14[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME _loop0_13")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_15[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_14")); + D(fprintf(stderr, "%*c%s _gather_14[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME _loop0_13")); } _res = NULL; done: @@ -28140,9 +28266,9 @@ _gather_15_rule(Parser *p) return _res; } -// _tmp_16: ';' | NEWLINE +// _tmp_15: ';' | NEWLINE static void * -_tmp_16_rule(Parser *p) +_tmp_15_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28158,18 +28284,18 @@ _tmp_16_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 13)) // token=';' ) { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); + D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';'")); } { // NEWLINE @@ -28177,18 +28303,18 @@ _tmp_16_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_15[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_15[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_15[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -28197,9 +28323,9 @@ _tmp_16_rule(Parser *p) return _res; } -// _tmp_17: ',' expression +// _tmp_16: ',' expression static void * -_tmp_17_rule(Parser *p) +_tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28215,7 +28341,7 @@ _tmp_17_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_16[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty z; if ( @@ -28224,7 +28350,7 @@ _tmp_17_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_17[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_16[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28234,7 +28360,7 @@ _tmp_17_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_17[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_16[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -28243,9 +28369,9 @@ _tmp_17_rule(Parser *p) return _res; } -// _loop0_18: ('.' | '...') +// _loop0_17: ('.' | '...') static asdl_seq * -_loop0_18_rule(Parser *p) +_loop0_17_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28270,13 +28396,13 @@ _loop0_18_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + D(fprintf(stderr, "%*c> _loop0_17[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_157_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_157_var = _tmp_157_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28293,7 +28419,7 @@ _loop0_18_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_18[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_17[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28310,9 +28436,9 @@ _loop0_18_rule(Parser *p) return _seq; } -// _loop1_19: ('.' | '...') +// _loop1_18: ('.' | '...') static asdl_seq * -_loop1_19_rule(Parser *p) +_loop1_18_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28337,13 +28463,13 @@ _loop1_19_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + D(fprintf(stderr, "%*c> _loop1_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); + void *_tmp_157_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_157_var = _tmp_157_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28360,7 +28486,7 @@ _loop1_19_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_19[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_18[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('.' | '...')")); } if (_n == 0 || p->error_indicator) { @@ -28382,9 +28508,9 @@ _loop1_19_rule(Parser *p) return _seq; } -// _loop0_20: ',' import_from_as_name +// _loop0_19: ',' import_from_as_name static asdl_seq * -_loop0_20_rule(Parser *p) +_loop0_19_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28409,7 +28535,7 @@ _loop0_20_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); + D(fprintf(stderr, "%*c> _loop0_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' import_from_as_name")); Token * _literal; alias_ty elem; while ( @@ -28441,7 +28567,7 @@ _loop0_20_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_20[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_19[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' import_from_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28458,9 +28584,9 @@ _loop0_20_rule(Parser *p) return _seq; } -// _gather_21: import_from_as_name _loop0_20 +// _gather_20: import_from_as_name _loop0_19 static asdl_seq * -_gather_21_rule(Parser *p) +_gather_20_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28471,27 +28597,27 @@ _gather_21_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // import_from_as_name _loop0_20 + { // import_from_as_name _loop0_19 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c> _gather_20[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_19")); alias_ty elem; asdl_seq * seq; if ( (elem = import_from_as_name_rule(p)) // import_from_as_name && - (seq = _loop0_20_rule(p)) // _loop0_20 + (seq = _loop0_19_rule(p)) // _loop0_19 ) { - D(fprintf(stderr, "%*c+ _gather_21[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c+ _gather_20[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_name _loop0_19")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_21[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_20")); + D(fprintf(stderr, "%*c%s _gather_20[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "import_from_as_name _loop0_19")); } _res = NULL; done: @@ -28499,9 +28625,9 @@ _gather_21_rule(Parser *p) return _res; } -// _tmp_22: 'as' NAME +// _tmp_21: 'as' NAME static void * -_tmp_22_rule(Parser *p) +_tmp_21_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28517,16 +28643,16 @@ _tmp_22_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_21[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (z = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_21[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28536,7 +28662,7 @@ _tmp_22_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_22[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_21[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -28545,9 +28671,9 @@ _tmp_22_rule(Parser *p) return _res; } -// _loop0_23: ',' dotted_as_name +// _loop0_22: ',' dotted_as_name static asdl_seq * -_loop0_23_rule(Parser *p) +_loop0_22_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28572,7 +28698,7 @@ _loop0_23_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); + D(fprintf(stderr, "%*c> _loop0_22[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_as_name")); Token * _literal; alias_ty elem; while ( @@ -28604,7 +28730,7 @@ _loop0_23_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_23[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_22[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_as_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28621,9 +28747,9 @@ _loop0_23_rule(Parser *p) return _seq; } -// _gather_24: dotted_as_name _loop0_23 +// _gather_23: dotted_as_name _loop0_22 static asdl_seq * -_gather_24_rule(Parser *p) +_gather_23_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28634,27 +28760,27 @@ _gather_24_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_as_name _loop0_23 + { // dotted_as_name _loop0_22 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c> _gather_23[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_22")); alias_ty elem; asdl_seq * seq; if ( (elem = dotted_as_name_rule(p)) // dotted_as_name && - (seq = _loop0_23_rule(p)) // _loop0_23 + (seq = _loop0_22_rule(p)) // _loop0_22 ) { - D(fprintf(stderr, "%*c+ _gather_24[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c+ _gather_23[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_as_name _loop0_22")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_24[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_23")); + D(fprintf(stderr, "%*c%s _gather_23[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_as_name _loop0_22")); } _res = NULL; done: @@ -28662,9 +28788,9 @@ _gather_24_rule(Parser *p) return _res; } -// _loop1_25: ('@' named_expression NEWLINE) +// _loop1_24: ('@' named_expression NEWLINE) static asdl_seq * -_loop1_25_rule(Parser *p) +_loop1_24_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28689,13 +28815,13 @@ _loop1_25_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_157_var; + D(fprintf(stderr, "%*c> _loop1_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); + void *_tmp_158_var; while ( - (_tmp_157_var = _tmp_157_rule(p)) // '@' named_expression NEWLINE + (_tmp_158_var = _tmp_158_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_157_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28712,7 +28838,7 @@ _loop1_25_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_25[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_24[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('@' named_expression NEWLINE)")); } if (_n == 0 || p->error_indicator) { @@ -28734,9 +28860,9 @@ _loop1_25_rule(Parser *p) return _seq; } -// _tmp_26: '(' arguments? ')' +// _tmp_25: '(' arguments? ')' static void * -_tmp_26_rule(Parser *p) +_tmp_25_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28752,7 +28878,7 @@ _tmp_26_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *z; @@ -28764,7 +28890,7 @@ _tmp_26_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_25[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28774,7 +28900,7 @@ _tmp_26_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_26[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_25[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -28783,9 +28909,9 @@ _tmp_26_rule(Parser *p) return _res; } -// _tmp_27: '->' expression +// _tmp_26: '->' expression static void * -_tmp_27_rule(Parser *p) +_tmp_26_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28801,7 +28927,7 @@ _tmp_27_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c> _tmp_26[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); Token * _literal; expr_ty z; if ( @@ -28810,7 +28936,7 @@ _tmp_27_rule(Parser *p) (z = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_27[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -28820,7 +28946,7 @@ _tmp_27_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_27[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_26[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); } _res = NULL; @@ -28829,9 +28955,9 @@ _tmp_27_rule(Parser *p) return _res; } -// _loop0_28: param_no_default +// _loop0_27: param_no_default static asdl_seq * -_loop0_28_rule(Parser *p) +_loop0_27_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28856,7 +28982,7 @@ _loop0_28_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop0_27[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -28879,7 +29005,7 @@ _loop0_28_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_28[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_27[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28896,9 +29022,9 @@ _loop0_28_rule(Parser *p) return _seq; } -// _loop0_29: param_with_default +// _loop0_28: param_with_default static asdl_seq * -_loop0_29_rule(Parser *p) +_loop0_28_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28923,7 +29049,7 @@ _loop0_29_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop0_28[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -28946,7 +29072,7 @@ _loop0_29_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_29[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_28[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -28963,9 +29089,9 @@ _loop0_29_rule(Parser *p) return _seq; } -// _loop1_30: param_no_default +// _loop1_29: param_no_default static asdl_seq * -_loop1_30_rule(Parser *p) +_loop1_29_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -28990,7 +29116,7 @@ _loop1_30_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _loop1_29[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; while ( (param_no_default_var = param_no_default_rule(p)) // param_no_default @@ -29013,7 +29139,7 @@ _loop1_30_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_30[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_29[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -29035,9 +29161,9 @@ _loop1_30_rule(Parser *p) return _seq; } -// _loop1_31: param_with_default +// _loop1_30: param_with_default static asdl_seq * -_loop1_31_rule(Parser *p) +_loop1_30_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29062,7 +29188,7 @@ _loop1_31_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); + D(fprintf(stderr, "%*c> _loop1_30[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_with_default")); NameDefaultPair* param_with_default_var; while ( (param_with_default_var = param_with_default_rule(p)) // param_with_default @@ -29085,7 +29211,7 @@ _loop1_31_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_31[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_30[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -29107,9 +29233,9 @@ _loop1_31_rule(Parser *p) return _seq; } -// _loop0_32: param_maybe_default +// _loop0_31: param_maybe_default static asdl_seq * -_loop0_32_rule(Parser *p) +_loop0_31_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29134,7 +29260,7 @@ _loop0_32_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_31[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -29157,7 +29283,7 @@ _loop0_32_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_32[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_31[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29174,9 +29300,9 @@ _loop0_32_rule(Parser *p) return _seq; } -// _loop1_33: param_maybe_default +// _loop1_32: param_maybe_default static asdl_seq * -_loop1_33_rule(Parser *p) +_loop1_32_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29201,7 +29327,7 @@ _loop1_33_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_maybe_default")); NameDefaultPair* param_maybe_default_var; while ( (param_maybe_default_var = param_maybe_default_rule(p)) // param_maybe_default @@ -29224,7 +29350,7 @@ _loop1_33_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_33[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_32[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -29246,9 +29372,9 @@ _loop1_33_rule(Parser *p) return _seq; } -// _loop0_34: ',' with_item +// _loop0_33: ',' with_item static asdl_seq * -_loop0_34_rule(Parser *p) +_loop0_33_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29273,7 +29399,7 @@ _loop0_34_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); + D(fprintf(stderr, "%*c> _loop0_33[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' with_item")); Token * _literal; withitem_ty elem; while ( @@ -29305,7 +29431,7 @@ _loop0_34_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_34[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_33[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' with_item")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29322,9 +29448,9 @@ _loop0_34_rule(Parser *p) return _seq; } -// _gather_35: with_item _loop0_34 +// _gather_34: with_item _loop0_33 static asdl_seq * -_gather_35_rule(Parser *p) +_gather_34_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29335,27 +29461,27 @@ _gather_35_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // with_item _loop0_34 + { // with_item _loop0_33 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c> _gather_34[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "with_item _loop0_33")); withitem_ty elem; asdl_seq * seq; if ( (elem = with_item_rule(p)) // with_item && - (seq = _loop0_34_rule(p)) // _loop0_34 + (seq = _loop0_33_rule(p)) // _loop0_33 ) { - D(fprintf(stderr, "%*c+ _gather_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c+ _gather_34[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "with_item _loop0_33")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_35[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_34")); + D(fprintf(stderr, "%*c%s _gather_34[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "with_item _loop0_33")); } _res = NULL; done: @@ -29363,9 +29489,9 @@ _gather_35_rule(Parser *p) return _res; } -// _tmp_36: ',' | ')' | ':' +// _tmp_35: ',' | ')' | ':' static void * -_tmp_36_rule(Parser *p) +_tmp_35_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29381,18 +29507,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -29400,18 +29526,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ':' @@ -29419,18 +29545,18 @@ _tmp_36_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_35[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_36[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_35[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_36[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_35[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -29439,9 +29565,9 @@ _tmp_36_rule(Parser *p) return _res; } -// _loop1_37: except_block +// _loop1_36: except_block static asdl_seq * -_loop1_37_rule(Parser *p) +_loop1_36_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29466,7 +29592,7 @@ _loop1_37_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); + D(fprintf(stderr, "%*c> _loop1_36[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_block")); excepthandler_ty except_block_var; while ( (except_block_var = except_block_rule(p)) // except_block @@ -29489,7 +29615,7 @@ _loop1_37_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_37[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_36[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_block")); } if (_n == 0 || p->error_indicator) { @@ -29511,9 +29637,9 @@ _loop1_37_rule(Parser *p) return _seq; } -// _loop1_38: except_star_block +// _loop1_37: except_star_block static asdl_seq * -_loop1_38_rule(Parser *p) +_loop1_37_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29538,7 +29664,7 @@ _loop1_38_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); + D(fprintf(stderr, "%*c> _loop1_37[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "except_star_block")); excepthandler_ty except_star_block_var; while ( (except_star_block_var = except_star_block_rule(p)) // except_star_block @@ -29561,7 +29687,7 @@ _loop1_38_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_38[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_37[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "except_star_block")); } if (_n == 0 || p->error_indicator) { @@ -29583,9 +29709,9 @@ _loop1_38_rule(Parser *p) return _seq; } -// _loop1_39: case_block +// _loop1_38: case_block static asdl_seq * -_loop1_39_rule(Parser *p) +_loop1_38_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29610,7 +29736,7 @@ _loop1_39_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); + D(fprintf(stderr, "%*c> _loop1_38[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "case_block")); match_case_ty case_block_var; while ( (case_block_var = case_block_rule(p)) // case_block @@ -29633,7 +29759,7 @@ _loop1_39_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_39[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_38[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "case_block")); } if (_n == 0 || p->error_indicator) { @@ -29655,9 +29781,9 @@ _loop1_39_rule(Parser *p) return _seq; } -// _loop0_40: '|' closed_pattern +// _loop0_39: '|' closed_pattern static asdl_seq * -_loop0_40_rule(Parser *p) +_loop0_39_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29682,7 +29808,7 @@ _loop0_40_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); + D(fprintf(stderr, "%*c> _loop0_39[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'|' closed_pattern")); Token * _literal; pattern_ty elem; while ( @@ -29714,7 +29840,7 @@ _loop0_40_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_40[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_39[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'|' closed_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -29731,9 +29857,9 @@ _loop0_40_rule(Parser *p) return _seq; } -// _gather_41: closed_pattern _loop0_40 +// _gather_40: closed_pattern _loop0_39 static asdl_seq * -_gather_41_rule(Parser *p) +_gather_40_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29744,27 +29870,27 @@ _gather_41_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // closed_pattern _loop0_40 + { // closed_pattern _loop0_39 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c> _gather_40[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_39")); pattern_ty elem; asdl_seq * seq; if ( (elem = closed_pattern_rule(p)) // closed_pattern && - (seq = _loop0_40_rule(p)) // _loop0_40 + (seq = _loop0_39_rule(p)) // _loop0_39 ) { - D(fprintf(stderr, "%*c+ _gather_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c+ _gather_40[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "closed_pattern _loop0_39")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_41[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_40")); + D(fprintf(stderr, "%*c%s _gather_40[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "closed_pattern _loop0_39")); } _res = NULL; done: @@ -29772,9 +29898,9 @@ _gather_41_rule(Parser *p) return _res; } -// _tmp_42: '+' | '-' +// _tmp_41: '+' | '-' static void * -_tmp_42_rule(Parser *p) +_tmp_41_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29790,18 +29916,18 @@ _tmp_42_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_41[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -29809,18 +29935,18 @@ _tmp_42_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_41[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_41[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_41[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } _res = NULL; @@ -29829,9 +29955,9 @@ _tmp_42_rule(Parser *p) return _res; } -// _tmp_43: STRING | FSTRING_START | TSTRING_START +// _tmp_42: STRING | FSTRING_START | TSTRING_START static void * -_tmp_43_rule(Parser *p) +_tmp_42_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29847,18 +29973,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "STRING")); expr_ty string_var; if ( (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "STRING")); } { // FSTRING_START @@ -29866,18 +29992,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); Token * fstring_start_var; if ( (fstring_start_var = _PyPegen_expect_token(p, FSTRING_START)) // token='FSTRING_START' ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START")); _res = fstring_start_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "FSTRING_START")); } { // TSTRING_START @@ -29885,18 +30011,18 @@ _tmp_43_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); + D(fprintf(stderr, "%*c> _tmp_42[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); Token * tstring_start_var; if ( (tstring_start_var = _PyPegen_expect_token(p, TSTRING_START)) // token='TSTRING_START' ) { - D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); + D(fprintf(stderr, "%*c+ _tmp_42[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_START")); _res = tstring_start_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_42[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "TSTRING_START")); } _res = NULL; @@ -29905,9 +30031,9 @@ _tmp_43_rule(Parser *p) return _res; } -// _tmp_44: '.' | '(' | '=' +// _tmp_43: '.' | '(' | '=' static void * -_tmp_44_rule(Parser *p) +_tmp_43_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -29923,18 +30049,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '(' @@ -29942,18 +30068,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '=' @@ -29961,18 +30087,18 @@ _tmp_44_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_43[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_44[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_43[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_44[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_43[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } _res = NULL; @@ -29981,9 +30107,9 @@ _tmp_44_rule(Parser *p) return _res; } -// _loop0_45: ',' maybe_star_pattern +// _loop0_44: ',' maybe_star_pattern static asdl_seq * -_loop0_45_rule(Parser *p) +_loop0_44_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30008,7 +30134,7 @@ _loop0_45_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); + D(fprintf(stderr, "%*c> _loop0_44[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' maybe_star_pattern")); Token * _literal; pattern_ty elem; while ( @@ -30040,7 +30166,7 @@ _loop0_45_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_45[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_44[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' maybe_star_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30057,9 +30183,9 @@ _loop0_45_rule(Parser *p) return _seq; } -// _gather_46: maybe_star_pattern _loop0_45 +// _gather_45: maybe_star_pattern _loop0_44 static asdl_seq * -_gather_46_rule(Parser *p) +_gather_45_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30070,27 +30196,27 @@ _gather_46_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // maybe_star_pattern _loop0_45 + { // maybe_star_pattern _loop0_44 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c> _gather_45[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); pattern_ty elem; asdl_seq * seq; if ( (elem = maybe_star_pattern_rule(p)) // maybe_star_pattern && - (seq = _loop0_45_rule(p)) // _loop0_45 + (seq = _loop0_44_rule(p)) // _loop0_44 ) { - D(fprintf(stderr, "%*c+ _gather_46[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c+ _gather_45[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern _loop0_44")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_46[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_45")); + D(fprintf(stderr, "%*c%s _gather_45[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "maybe_star_pattern _loop0_44")); } _res = NULL; done: @@ -30098,9 +30224,9 @@ _gather_46_rule(Parser *p) return _res; } -// _loop0_47: ',' key_value_pattern +// _loop0_46: ',' key_value_pattern static asdl_seq * -_loop0_47_rule(Parser *p) +_loop0_46_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30125,7 +30251,7 @@ _loop0_47_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); + D(fprintf(stderr, "%*c> _loop0_46[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' key_value_pattern")); Token * _literal; KeyPatternPair* elem; while ( @@ -30157,7 +30283,7 @@ _loop0_47_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_47[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_46[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' key_value_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30174,9 +30300,9 @@ _loop0_47_rule(Parser *p) return _seq; } -// _gather_48: key_value_pattern _loop0_47 +// _gather_47: key_value_pattern _loop0_46 static asdl_seq * -_gather_48_rule(Parser *p) +_gather_47_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30187,27 +30313,27 @@ _gather_48_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // key_value_pattern _loop0_47 + { // key_value_pattern _loop0_46 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c> _gather_47[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); KeyPatternPair* elem; asdl_seq * seq; if ( (elem = key_value_pattern_rule(p)) // key_value_pattern && - (seq = _loop0_47_rule(p)) // _loop0_47 + (seq = _loop0_46_rule(p)) // _loop0_46 ) { - D(fprintf(stderr, "%*c+ _gather_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c+ _gather_47[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "key_value_pattern _loop0_46")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_48[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_47")); + D(fprintf(stderr, "%*c%s _gather_47[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "key_value_pattern _loop0_46")); } _res = NULL; done: @@ -30215,9 +30341,9 @@ _gather_48_rule(Parser *p) return _res; } -// _tmp_49: literal_expr | attr +// _tmp_48: literal_expr | attr static void * -_tmp_49_rule(Parser *p) +_tmp_48_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30233,18 +30359,18 @@ _tmp_49_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "literal_expr")); expr_ty literal_expr_var; if ( (literal_expr_var = literal_expr_rule(p)) // literal_expr ) { - D(fprintf(stderr, "%*c+ _tmp_49[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "literal_expr")); _res = literal_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_49[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "literal_expr")); } { // attr @@ -30252,18 +30378,18 @@ _tmp_49_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); + D(fprintf(stderr, "%*c> _tmp_48[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "attr")); expr_ty attr_var; if ( (attr_var = attr_rule(p)) // attr ) { - D(fprintf(stderr, "%*c+ _tmp_49[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); + D(fprintf(stderr, "%*c+ _tmp_48[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr")); _res = attr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_49[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_48[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "attr")); } _res = NULL; @@ -30272,9 +30398,9 @@ _tmp_49_rule(Parser *p) return _res; } -// _loop0_50: ',' pattern +// _loop0_49: ',' pattern static asdl_seq * -_loop0_50_rule(Parser *p) +_loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30299,7 +30425,7 @@ _loop0_50_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); + D(fprintf(stderr, "%*c> _loop0_49[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' pattern")); Token * _literal; pattern_ty elem; while ( @@ -30331,7 +30457,7 @@ _loop0_50_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_50[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_49[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30348,9 +30474,9 @@ _loop0_50_rule(Parser *p) return _seq; } -// _gather_51: pattern _loop0_50 +// _gather_50: pattern _loop0_49 static asdl_seq * -_gather_51_rule(Parser *p) +_gather_50_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30361,27 +30487,27 @@ _gather_51_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // pattern _loop0_50 + { // pattern _loop0_49 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c> _gather_50[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); pattern_ty elem; asdl_seq * seq; if ( (elem = pattern_rule(p)) // pattern && - (seq = _loop0_50_rule(p)) // _loop0_50 + (seq = _loop0_49_rule(p)) // _loop0_49 ) { - D(fprintf(stderr, "%*c+ _gather_51[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c+ _gather_50[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pattern _loop0_49")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_51[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_50")); + D(fprintf(stderr, "%*c%s _gather_50[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pattern _loop0_49")); } _res = NULL; done: @@ -30389,9 +30515,9 @@ _gather_51_rule(Parser *p) return _res; } -// _loop0_52: ',' keyword_pattern +// _loop0_51: ',' keyword_pattern static asdl_seq * -_loop0_52_rule(Parser *p) +_loop0_51_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30416,7 +30542,7 @@ _loop0_52_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); + D(fprintf(stderr, "%*c> _loop0_51[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' keyword_pattern")); Token * _literal; KeyPatternPair* elem; while ( @@ -30448,7 +30574,7 @@ _loop0_52_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_52[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_51[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' keyword_pattern")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30465,9 +30591,9 @@ _loop0_52_rule(Parser *p) return _seq; } -// _gather_53: keyword_pattern _loop0_52 +// _gather_52: keyword_pattern _loop0_51 static asdl_seq * -_gather_53_rule(Parser *p) +_gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30478,27 +30604,27 @@ _gather_53_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // keyword_pattern _loop0_52 + { // keyword_pattern _loop0_51 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c> _gather_52[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); KeyPatternPair* elem; asdl_seq * seq; if ( (elem = keyword_pattern_rule(p)) // keyword_pattern && - (seq = _loop0_52_rule(p)) // _loop0_52 + (seq = _loop0_51_rule(p)) // _loop0_51 ) { - D(fprintf(stderr, "%*c+ _gather_53[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c+ _gather_52[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "keyword_pattern _loop0_51")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_53[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_52")); + D(fprintf(stderr, "%*c%s _gather_52[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "keyword_pattern _loop0_51")); } _res = NULL; done: @@ -30506,9 +30632,9 @@ _gather_53_rule(Parser *p) return _res; } -// _loop0_54: ',' type_param +// _loop0_53: ',' type_param static asdl_seq * -_loop0_54_rule(Parser *p) +_loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30533,7 +30659,7 @@ _loop0_54_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); + D(fprintf(stderr, "%*c> _loop0_53[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' type_param")); Token * _literal; type_param_ty elem; while ( @@ -30565,7 +30691,7 @@ _loop0_54_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_54[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_53[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' type_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30582,9 +30708,9 @@ _loop0_54_rule(Parser *p) return _seq; } -// _gather_55: type_param _loop0_54 +// _gather_54: type_param _loop0_53 static asdl_seq * -_gather_55_rule(Parser *p) +_gather_54_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30595,27 +30721,27 @@ _gather_55_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // type_param _loop0_54 + { // type_param _loop0_53 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c> _gather_54[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); type_param_ty elem; asdl_seq * seq; if ( (elem = type_param_rule(p)) // type_param && - (seq = _loop0_54_rule(p)) // _loop0_54 + (seq = _loop0_53_rule(p)) // _loop0_53 ) { - D(fprintf(stderr, "%*c+ _gather_55[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c+ _gather_54[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "type_param _loop0_53")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_55[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_54")); + D(fprintf(stderr, "%*c%s _gather_54[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "type_param _loop0_53")); } _res = NULL; done: @@ -30623,9 +30749,9 @@ _gather_55_rule(Parser *p) return _res; } -// _loop1_56: (',' expression) +// _loop1_55: (',' expression) static asdl_seq * -_loop1_56_rule(Parser *p) +_loop1_55_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30650,13 +30776,13 @@ _loop1_56_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_17_var; + D(fprintf(stderr, "%*c> _loop1_55[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); + void *_tmp_16_var; while ( - (_tmp_17_var = _tmp_17_rule(p)) // ',' expression + (_tmp_16_var = _tmp_16_rule(p)) // ',' expression ) { - _res = _tmp_17_var; + _res = _tmp_16_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30673,7 +30799,7 @@ _loop1_56_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_56[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_55[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' expression)")); } if (_n == 0 || p->error_indicator) { @@ -30695,9 +30821,9 @@ _loop1_56_rule(Parser *p) return _seq; } -// _loop1_57: (',' star_expression) +// _loop1_56: (',' star_expression) static asdl_seq * -_loop1_57_rule(Parser *p) +_loop1_56_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30722,13 +30848,13 @@ _loop1_57_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_158_var; + D(fprintf(stderr, "%*c> _loop1_56[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); + void *_tmp_159_var; while ( - (_tmp_158_var = _tmp_158_rule(p)) // ',' star_expression + (_tmp_159_var = _tmp_159_rule(p)) // ',' star_expression ) { - _res = _tmp_158_var; + _res = _tmp_159_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30745,7 +30871,7 @@ _loop1_57_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_57[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_56[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_expression)")); } if (_n == 0 || p->error_indicator) { @@ -30767,9 +30893,9 @@ _loop1_57_rule(Parser *p) return _seq; } -// _loop0_58: ',' star_named_expression +// _loop0_57: ',' star_named_expression static asdl_seq * -_loop0_58_rule(Parser *p) +_loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30794,7 +30920,7 @@ _loop0_58_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); + D(fprintf(stderr, "%*c> _loop0_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_named_expression")); Token * _literal; expr_ty elem; while ( @@ -30826,7 +30952,7 @@ _loop0_58_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_58[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_57[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_named_expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -30843,9 +30969,9 @@ _loop0_58_rule(Parser *p) return _seq; } -// _gather_59: star_named_expression _loop0_58 +// _gather_58: star_named_expression _loop0_57 static asdl_seq * -_gather_59_rule(Parser *p) +_gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30856,27 +30982,27 @@ _gather_59_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_named_expression _loop0_58 + { // star_named_expression _loop0_57 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c> _gather_58[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); expr_ty elem; asdl_seq * seq; if ( (elem = star_named_expression_rule(p)) // star_named_expression && - (seq = _loop0_58_rule(p)) // _loop0_58 + (seq = _loop0_57_rule(p)) // _loop0_57 ) { - D(fprintf(stderr, "%*c+ _gather_59[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c+ _gather_58[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression _loop0_57")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_59[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_58")); + D(fprintf(stderr, "%*c%s _gather_58[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression _loop0_57")); } _res = NULL; done: @@ -30884,9 +31010,9 @@ _gather_59_rule(Parser *p) return _res; } -// _loop1_60: ('or' conjunction) +// _loop1_59: ('or' conjunction) static asdl_seq * -_loop1_60_rule(Parser *p) +_loop1_59_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30911,13 +31037,13 @@ _loop1_60_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_159_var; + D(fprintf(stderr, "%*c> _loop1_59[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); + void *_tmp_160_var; while ( - (_tmp_159_var = _tmp_159_rule(p)) // 'or' conjunction + (_tmp_160_var = _tmp_160_rule(p)) // 'or' conjunction ) { - _res = _tmp_159_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30934,7 +31060,7 @@ _loop1_60_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_59[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('or' conjunction)")); } if (_n == 0 || p->error_indicator) { @@ -30956,9 +31082,9 @@ _loop1_60_rule(Parser *p) return _seq; } -// _loop1_61: ('and' inversion) +// _loop1_60: ('and' inversion) static asdl_seq * -_loop1_61_rule(Parser *p) +_loop1_60_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -30983,13 +31109,13 @@ _loop1_61_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_160_var; + D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); + void *_tmp_161_var; while ( - (_tmp_160_var = _tmp_160_rule(p)) // 'and' inversion + (_tmp_161_var = _tmp_161_rule(p)) // 'and' inversion ) { - _res = _tmp_160_var; + _res = _tmp_161_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31006,7 +31132,7 @@ _loop1_61_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_60[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('and' inversion)")); } if (_n == 0 || p->error_indicator) { @@ -31028,9 +31154,9 @@ _loop1_61_rule(Parser *p) return _seq; } -// _loop1_62: compare_op_bitwise_or_pair +// _loop1_61: compare_op_bitwise_or_pair static asdl_seq * -_loop1_62_rule(Parser *p) +_loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31055,7 +31181,7 @@ _loop1_62_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); + D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "compare_op_bitwise_or_pair")); CmpopExprPair* compare_op_bitwise_or_pair_var; while ( (compare_op_bitwise_or_pair_var = compare_op_bitwise_or_pair_rule(p)) // compare_op_bitwise_or_pair @@ -31078,7 +31204,7 @@ _loop1_62_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_62[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_61[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "compare_op_bitwise_or_pair")); } if (_n == 0 || p->error_indicator) { @@ -31100,9 +31226,9 @@ _loop1_62_rule(Parser *p) return _seq; } -// _tmp_63: '!=' +// _tmp_62: '!=' static void * -_tmp_63_rule(Parser *p) +_tmp_62_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31118,13 +31244,13 @@ _tmp_63_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); + D(fprintf(stderr, "%*c> _tmp_62[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!='")); Token * tok; if ( (tok = _PyPegen_expect_token(p, 28)) // token='!=' ) { - D(fprintf(stderr, "%*c+ _tmp_63[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); + D(fprintf(stderr, "%*c+ _tmp_62[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); _res = _PyPegen_check_barry_as_flufl ( p , tok ) ? NULL : tok; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -31134,7 +31260,7 @@ _tmp_63_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_63[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_62[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!='")); } _res = NULL; @@ -31143,9 +31269,9 @@ _tmp_63_rule(Parser *p) return _res; } -// _loop0_64: ',' (slice | starred_expression) +// _loop0_63: ',' (slice | starred_expression) static asdl_seq * -_loop0_64_rule(Parser *p) +_loop0_63_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31170,13 +31296,13 @@ _loop0_64_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); + D(fprintf(stderr, "%*c> _loop0_63[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (slice | starred_expression)")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_162_rule(p)) // slice | starred_expression ) { _res = elem; @@ -31202,7 +31328,7 @@ _loop0_64_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_64[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_63[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (slice | starred_expression)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31219,9 +31345,9 @@ _loop0_64_rule(Parser *p) return _seq; } -// _gather_65: (slice | starred_expression) _loop0_64 +// _gather_64: (slice | starred_expression) _loop0_63 static asdl_seq * -_gather_65_rule(Parser *p) +_gather_64_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31232,27 +31358,27 @@ _gather_65_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (slice | starred_expression) _loop0_64 + { // (slice | starred_expression) _loop0_63 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c> _gather_64[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_162_rule(p)) // slice | starred_expression && - (seq = _loop0_64_rule(p)) // _loop0_64 + (seq = _loop0_63_rule(p)) // _loop0_63 ) { - D(fprintf(stderr, "%*c+ _gather_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c+ _gather_64[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slice | starred_expression) _loop0_63")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_65[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_64")); + D(fprintf(stderr, "%*c%s _gather_64[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(slice | starred_expression) _loop0_63")); } _res = NULL; done: @@ -31260,9 +31386,9 @@ _gather_65_rule(Parser *p) return _res; } -// _tmp_66: ':' expression? +// _tmp_65: ':' expression? static void * -_tmp_66_rule(Parser *p) +_tmp_65_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31278,7 +31404,7 @@ _tmp_66_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); + D(fprintf(stderr, "%*c> _tmp_65[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' expression?")); Token * _literal; void *d; if ( @@ -31287,7 +31413,7 @@ _tmp_66_rule(Parser *p) (d = expression_rule(p), !p->error_indicator) // expression? ) { - D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); + D(fprintf(stderr, "%*c+ _tmp_65[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); _res = d; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -31297,7 +31423,7 @@ _tmp_66_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_65[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':' expression?")); } _res = NULL; @@ -31306,9 +31432,9 @@ _tmp_66_rule(Parser *p) return _res; } -// _tmp_67: tuple | group | genexp +// _tmp_66: tuple | group | genexp static void * -_tmp_67_rule(Parser *p) +_tmp_66_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31324,18 +31450,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // group @@ -31343,18 +31469,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "group")); expr_ty group_var; if ( (group_var = group_rule(p)) // group ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "group")); _res = group_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "group")); } { // genexp @@ -31362,18 +31488,18 @@ _tmp_67_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_66[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_66[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } _res = NULL; @@ -31382,9 +31508,9 @@ _tmp_67_rule(Parser *p) return _res; } -// _tmp_68: list | listcomp +// _tmp_67: list | listcomp static void * -_tmp_68_rule(Parser *p) +_tmp_67_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31400,18 +31526,18 @@ _tmp_68_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // listcomp @@ -31419,18 +31545,18 @@ _tmp_68_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); + D(fprintf(stderr, "%*c> _tmp_67[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "listcomp")); expr_ty listcomp_var; if ( (listcomp_var = listcomp_rule(p)) // listcomp ) { - D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); + D(fprintf(stderr, "%*c+ _tmp_67[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "listcomp")); _res = listcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_67[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "listcomp")); } _res = NULL; @@ -31439,9 +31565,9 @@ _tmp_68_rule(Parser *p) return _res; } -// _tmp_69: dict | set | dictcomp | setcomp +// _tmp_68: dict | set | dictcomp | setcomp static void * -_tmp_69_rule(Parser *p) +_tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31457,18 +31583,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dict")); expr_ty dict_var; if ( (dict_var = dict_rule(p)) // dict ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dict")); _res = dict_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dict")); } { // set @@ -31476,18 +31602,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "set")); expr_ty set_var; if ( (set_var = set_rule(p)) // set ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "set")); _res = set_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "set")); } { // dictcomp @@ -31495,18 +31621,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dictcomp")); expr_ty dictcomp_var; if ( (dictcomp_var = dictcomp_rule(p)) // dictcomp ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dictcomp")); _res = dictcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dictcomp")); } { // setcomp @@ -31514,18 +31640,18 @@ _tmp_69_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); + D(fprintf(stderr, "%*c> _tmp_68[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "setcomp")); expr_ty setcomp_var; if ( (setcomp_var = setcomp_rule(p)) // setcomp ) { - D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); + D(fprintf(stderr, "%*c+ _tmp_68[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "setcomp")); _res = setcomp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_68[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "setcomp")); } _res = NULL; @@ -31534,9 +31660,9 @@ _tmp_69_rule(Parser *p) return _res; } -// _tmp_70: yield_expr | named_expression +// _tmp_69: yield_expr | named_expression static void * -_tmp_70_rule(Parser *p) +_tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31552,18 +31678,18 @@ _tmp_70_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "yield_expr")); expr_ty yield_expr_var; if ( (yield_expr_var = yield_expr_rule(p)) // yield_expr ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "yield_expr")); _res = yield_expr_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "yield_expr")); } { // named_expression @@ -31571,18 +31697,18 @@ _tmp_70_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); + D(fprintf(stderr, "%*c> _tmp_69[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "named_expression")); expr_ty named_expression_var; if ( (named_expression_var = named_expression_rule(p)) // named_expression ) { - D(fprintf(stderr, "%*c+ _tmp_70[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); + D(fprintf(stderr, "%*c+ _tmp_69[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); _res = named_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_70[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_69[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "named_expression")); } _res = NULL; @@ -31591,9 +31717,9 @@ _tmp_70_rule(Parser *p) return _res; } -// _loop0_71: lambda_param_no_default +// _loop0_70: lambda_param_no_default static asdl_seq * -_loop0_71_rule(Parser *p) +_loop0_70_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31618,7 +31744,7 @@ _loop0_71_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop0_70[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -31641,7 +31767,7 @@ _loop0_71_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_71[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_70[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31658,9 +31784,9 @@ _loop0_71_rule(Parser *p) return _seq; } -// _loop0_72: lambda_param_with_default +// _loop0_71: lambda_param_with_default static asdl_seq * -_loop0_72_rule(Parser *p) +_loop0_71_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31685,7 +31811,7 @@ _loop0_72_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop0_71[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -31708,7 +31834,7 @@ _loop0_72_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_72[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_71[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31725,9 +31851,9 @@ _loop0_72_rule(Parser *p) return _seq; } -// _loop1_73: lambda_param_no_default +// _loop1_72: lambda_param_no_default static asdl_seq * -_loop1_73_rule(Parser *p) +_loop1_72_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31752,7 +31878,7 @@ _loop1_73_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _loop1_72[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; while ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default @@ -31775,7 +31901,7 @@ _loop1_73_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_73[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_72[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } if (_n == 0 || p->error_indicator) { @@ -31797,9 +31923,9 @@ _loop1_73_rule(Parser *p) return _seq; } -// _loop1_74: lambda_param_with_default +// _loop1_73: lambda_param_with_default static asdl_seq * -_loop1_74_rule(Parser *p) +_loop1_73_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31824,7 +31950,7 @@ _loop1_74_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); + D(fprintf(stderr, "%*c> _loop1_73[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default")); NameDefaultPair* lambda_param_with_default_var; while ( (lambda_param_with_default_var = lambda_param_with_default_rule(p)) // lambda_param_with_default @@ -31847,7 +31973,7 @@ _loop1_74_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_74[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_73[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_with_default")); } if (_n == 0 || p->error_indicator) { @@ -31869,9 +31995,9 @@ _loop1_74_rule(Parser *p) return _seq; } -// _loop0_75: lambda_param_maybe_default +// _loop0_74: lambda_param_maybe_default static asdl_seq * -_loop0_75_rule(Parser *p) +_loop0_74_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31896,7 +32022,7 @@ _loop0_75_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop0_74[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -31919,7 +32045,7 @@ _loop0_75_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_75[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_74[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -31936,9 +32062,9 @@ _loop0_75_rule(Parser *p) return _seq; } -// _loop1_76: lambda_param_maybe_default +// _loop1_75: lambda_param_maybe_default static asdl_seq * -_loop1_76_rule(Parser *p) +_loop1_75_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -31963,7 +32089,7 @@ _loop1_76_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); + D(fprintf(stderr, "%*c> _loop1_75[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default")); NameDefaultPair* lambda_param_maybe_default_var; while ( (lambda_param_maybe_default_var = lambda_param_maybe_default_rule(p)) // lambda_param_maybe_default @@ -31986,7 +32112,7 @@ _loop1_76_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_76[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_75[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_maybe_default")); } if (_n == 0 || p->error_indicator) { @@ -32008,9 +32134,9 @@ _loop1_76_rule(Parser *p) return _seq; } -// _loop0_77: fstring_format_spec +// _loop0_76: fstring_format_spec static asdl_seq * -_loop0_77_rule(Parser *p) +_loop0_76_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32035,7 +32161,7 @@ _loop0_77_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_76[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -32058,7 +32184,7 @@ _loop0_77_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_76[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32075,9 +32201,9 @@ _loop0_77_rule(Parser *p) return _seq; } -// _loop0_78: fstring_middle +// _loop0_77: fstring_middle static asdl_seq * -_loop0_78_rule(Parser *p) +_loop0_77_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32102,7 +32228,7 @@ _loop0_78_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); + D(fprintf(stderr, "%*c> _loop0_77[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_middle")); expr_ty fstring_middle_var; while ( (fstring_middle_var = fstring_middle_rule(p)) // fstring_middle @@ -32125,7 +32251,7 @@ _loop0_78_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_78[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_77[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_middle")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32142,9 +32268,9 @@ _loop0_78_rule(Parser *p) return _seq; } -// _loop0_79: tstring_format_spec +// _loop0_78: tstring_format_spec static asdl_seq * -_loop0_79_rule(Parser *p) +_loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32169,7 +32295,7 @@ _loop0_79_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_78[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_format_spec")); expr_ty tstring_format_spec_var; while ( (tstring_format_spec_var = tstring_format_spec_rule(p)) // tstring_format_spec @@ -32192,7 +32318,7 @@ _loop0_79_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_79[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_78[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32209,9 +32335,9 @@ _loop0_79_rule(Parser *p) return _seq; } -// _loop0_80: tstring_middle +// _loop0_79: tstring_middle static asdl_seq * -_loop0_80_rule(Parser *p) +_loop0_79_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32236,7 +32362,7 @@ _loop0_80_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_middle")); + D(fprintf(stderr, "%*c> _loop0_79[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring_middle")); expr_ty tstring_middle_var; while ( (tstring_middle_var = tstring_middle_rule(p)) // tstring_middle @@ -32259,7 +32385,7 @@ _loop0_80_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_80[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_79[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring_middle")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32276,7 +32402,79 @@ _loop0_80_rule(Parser *p) return _seq; } -// _loop1_81: (fstring | string | tstring) +// _loop1_80: (fstring | string) +static asdl_seq * +_loop1_80_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_80[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_153_var; + while ( + (_tmp_153_var = _tmp_153_rule(p)) // fstring | string + ) + { + _res = _tmp_153_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_80[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_81: tstring static asdl_seq * _loop1_81_rule(Parser *p) { @@ -32298,18 +32496,18 @@ _loop1_81_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (fstring | string | tstring) + { // tstring if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string | tstring)")); - void *_tmp_162_var; + D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); + expr_ty tstring_var; while ( - (_tmp_162_var = _tmp_162_rule(p)) // fstring | string | tstring + (tstring_var = tstring_rule(p)) // tstring ) { - _res = _tmp_162_var; + _res = tstring_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32327,7 +32525,7 @@ _loop1_81_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_81[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string | tstring)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -34036,7 +34234,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -34055,7 +34253,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -34074,7 +34272,7 @@ _tmp_111_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -34285,7 +34483,7 @@ _tmp_115_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 690)) // token='else' + (_keyword = _PyPegen_expect_token(p, 691)) // token='else' ) { D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); @@ -34532,7 +34730,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 622)) // token='True' + (_keyword = _PyPegen_expect_token(p, 623)) // token='True' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); @@ -34551,7 +34749,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 623)) // token='None' + (_keyword = _PyPegen_expect_token(p, 624)) // token='None' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); @@ -34570,7 +34768,7 @@ _tmp_118_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 624)) // token='False' + (_keyword = _PyPegen_expect_token(p, 625)) // token='False' ) { D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); @@ -34682,12 +34880,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_156_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_156_var = _tmp_156_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_156_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -36024,7 +36222,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 681)) // token='except' + (_keyword = _PyPegen_expect_token(p, 682)) // token='except' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); @@ -36043,7 +36241,7 @@ _tmp_143_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='finally' + (_keyword = _PyPegen_expect_token(p, 678)) // token='finally' ) { D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); @@ -36152,7 +36350,7 @@ _tmp_145_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_21_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); @@ -36593,9 +36791,66 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_153: fstring | string static void * _tmp_153_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // fstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + expr_ty fstring_var; + if ( + (fstring_var = fstring_rule(p)) // fstring + ) + { + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + _res = fstring_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); + } + { // string + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + expr_ty string_var; + if ( + (string_var = string_rule(p)) // string + ) + { + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + _res = string_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_154: '+' | '-' | '*' | '/' | '%' | '//' | '@' +static void * +_tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36611,18 +36866,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36630,18 +36885,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -36649,18 +36904,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -36668,18 +36923,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -36687,18 +36942,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -36706,18 +36961,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -36725,18 +36980,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -36745,9 +37000,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: '+' | '-' | '~' +// _tmp_155: '+' | '-' | '~' static void * -_tmp_154_rule(Parser *p) +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36763,18 +37018,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36782,18 +37037,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -36801,18 +37056,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -36821,9 +37076,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: star_targets '=' +// _tmp_156: star_targets '=' static void * -_tmp_155_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36839,7 +37094,7 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -36848,7 +37103,7 @@ _tmp_155_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36858,7 +37113,7 @@ _tmp_155_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -36867,9 +37122,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: '.' | '...' +// _tmp_157: '.' | '...' static void * -_tmp_156_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36885,18 +37140,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -36904,18 +37159,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -36924,9 +37179,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: '@' named_expression NEWLINE +// _tmp_158: '@' named_expression NEWLINE static void * -_tmp_157_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36942,7 +37197,7 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -36954,7 +37209,7 @@ _tmp_157_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36964,7 +37219,7 @@ _tmp_157_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -36973,9 +37228,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: ',' star_expression +// _tmp_159: ',' star_expression static void * -_tmp_158_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36991,7 +37246,7 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -37000,7 +37255,7 @@ _tmp_158_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37010,7 +37265,7 @@ _tmp_158_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -37019,9 +37274,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: 'or' conjunction +// _tmp_160: 'or' conjunction static void * -_tmp_159_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37037,16 +37292,16 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 588)) // token='or' + (_keyword = _PyPegen_expect_token(p, 589)) // token='or' && (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37056,7 +37311,7 @@ _tmp_159_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -37065,9 +37320,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: 'and' inversion +// _tmp_161: 'and' inversion static void * -_tmp_160_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37083,16 +37338,16 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( - (_keyword = _PyPegen_expect_token(p, 589)) // token='and' + (_keyword = _PyPegen_expect_token(p, 590)) // token='and' && (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37102,7 +37357,7 @@ _tmp_160_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -37111,9 +37366,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _tmp_161: slice | starred_expression +// _tmp_162: slice | starred_expression static void * -_tmp_161_rule(Parser *p) +_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37129,18 +37384,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -37148,95 +37403,19 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_162: fstring | string | tstring -static void * -_tmp_162_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // fstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); - expr_ty fstring_var; - if ( - (fstring_var = fstring_rule(p)) // fstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); - _res = fstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); - } - { // string - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); - expr_ty string_var; - if ( - (string_var = string_rule(p)) // string - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); - _res = string_var; - goto done; - } - p->mark = _mark; D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); - } - { // tstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); - expr_ty tstring_var; - if ( - (tstring_var = tstring_rule(p)) // tstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring")); - _res = tstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; done: @@ -37266,7 +37445,7 @@ _tmp_163_rule(Parser *p) Token * _keyword; expr_ty z; if ( - (_keyword = _PyPegen_expect_token(p, 686)) // token='if' + (_keyword = _PyPegen_expect_token(p, 687)) // token='if' && (z = disjunction_rule(p)) // disjunction ) @@ -37924,7 +38103,7 @@ _tmp_176_rule(Parser *p) Token * _keyword; expr_ty star_target_var; if ( - (_keyword = _PyPegen_expect_token(p, 684)) // token='as' + (_keyword = _PyPegen_expect_token(p, 685)) // token='as' && (star_target_var = star_target_rule(p)) // star_target ) diff --git a/Parser/pegen.h b/Parser/pegen.h index 804f931871aec8..dfa2b0b4fe726c 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -351,6 +351,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok); +expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int); expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 577da65c7cdafa..28c004c3c5ac2c 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -379,9 +379,9 @@ static int test_pre_initialization_sys_options(void) /* bpo-20891: Avoid race condition when initialising the GIL */ -static void bpo20891_thread(void *lockp) +static void bpo20891_thread(void *eventp) { - PyThread_type_lock lock = *((PyThread_type_lock*)lockp); + PyEvent *event = (PyEvent *)eventp; PyGILState_STATE state = PyGILState_Ensure(); if (!PyGILState_Check()) { @@ -390,8 +390,7 @@ static void bpo20891_thread(void *lockp) } PyGILState_Release(state); - - PyThread_release_lock(lock); + _PyEvent_Notify(event); } static int test_bpo20891(void) @@ -401,27 +400,16 @@ static int test_bpo20891(void) /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not crash. */ - PyThread_type_lock lock = PyThread_allocate_lock(); - if (!lock) { - error("PyThread_allocate_lock failed!"); - return 1; - } - + PyEvent event = {0}; _testembed_initialize(); - unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); + unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &event); if (thrd == PYTHREAD_INVALID_THREAD_ID) { error("PyThread_start_new_thread failed!"); return 1; } - PyThread_acquire_lock(lock, WAIT_LOCK); - - Py_BEGIN_ALLOW_THREADS - /* wait until the thread exit */ - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_END_ALLOW_THREADS - PyThread_free_lock(lock); + PyEvent_Wait(&event); Py_Finalize(); @@ -1854,9 +1842,9 @@ static int test_initconfig_get_api(void) assert(initconfig_getint(config, "dev_mode") == 1); // test PyInitConfig_GetInt() on a PyPreConfig option - assert(initconfig_getint(config, "utf8_mode") == 0); - assert(PyInitConfig_SetInt(config, "utf8_mode", 1) == 0); assert(initconfig_getint(config, "utf8_mode") == 1); + assert(PyInitConfig_SetInt(config, "utf8_mode", 0) == 0); + assert(initconfig_getint(config, "utf8_mode") == 0); // test PyInitConfig_GetStr() char *str; diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 616752459ba4d9..93adae3d99038f 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,5 +1,9 @@ .text +#if defined(__APPLE__) + .globl __Py_trampoline_func_start +#else .globl _Py_trampoline_func_start +#endif # The following assembly is equivalent to: # PyObject * # trampoline(PyThreadState *ts, _PyInterpreterFrame *f, @@ -7,14 +11,19 @@ # { # return evaluator(ts, f, throwflag); # } +#if defined(__APPLE__) +__Py_trampoline_func_start: +#else _Py_trampoline_func_start: +#endif #ifdef __x86_64__ #if defined(__CET__) && (__CET__ & 1) endbr64 #endif - sub $8, %rsp - call *%rcx - add $8, %rsp + push %rbp + mov %rsp, %rbp + call *%rcx + pop %rbp ret #endif // __x86_64__ #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) @@ -34,9 +43,14 @@ _Py_trampoline_func_start: addi sp,sp,16 jr ra #endif +#if defined(__APPLE__) + .globl __Py_trampoline_func_end +__Py_trampoline_func_end: +#else .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +#endif # Note for indicating the assembly code supports CET #if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) .section .note.gnu.property,"a" diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 557c12cfda61ff..c25699978cf651 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec) } static int -_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx, - Py_ssize_t last_idx, char prefix, PyArena *arena) +append_templatestr(PyUnicodeWriter *writer, expr_ty e) { int result = -1; - - asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena); - if (!new_values) { - return result; - } - - Py_ssize_t j = 0; - for (Py_ssize_t i = first_idx; i <= last_idx; ++i) { - asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i)); - } - - PyObject *body = build_ftstring_body(new_values, false); + PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false); if (!body) { - return result; + return -1; } - if (-1 != append_char(writer, prefix) && + if (-1 != append_charp(writer, "t") && -1 != append_repr(writer, body)) { result = 0; @@ -692,72 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_ return result; } -static int -append_templatestr(PyUnicodeWriter *writer, expr_ty e) -{ - PyArena *arena = _PyArena_New(); - if (!arena) { - return -1; - } - - Py_ssize_t last_idx = 0; - Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values); - if (len == 0) { - int result = _write_values_subarray(writer, e->v.TemplateStr.values, - 0, len - 1, 't', arena); - _PyArena_Free(arena); - return result; - } - - for (Py_ssize_t i = 0; i < len; i++) { - expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i); - - // Handle implicit concat of t-strings with f-strings - if (value->kind == FormattedValue_kind) { - if (i > last_idx) { - // Create a new TemplateStr with the values between last_idx and i - // and append it to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, i - 1, 't', arena) == -1) { - goto error; - } - - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - // Append the FormattedValue to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - i, i, 'f', arena) == -1) { - goto error; - } - - if (i + 1 < len) { - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - last_idx = i + 1; - } - } - - if (last_idx < len) { - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, len - 1, 't', arena) == -1) { - goto error; - } - } - _PyArena_Free(arena); - - return 0; - -error: - _PyArena_Free(arena); - return -1; -} - static int append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1a5a9ff13a23a5..d9abc4c53d1f50 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2327,19 +2327,18 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + *attr = PyStackRef_NULL; + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(attr_o != NULL); // No errors on this branch + assert(!PyStackRef_IsNull(*attr)); // No errors on this branch self_or_null[0] = owner; // Transfer ownership DEAD(owner); } @@ -2351,17 +2350,17 @@ dummy_func( meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - ERROR_IF(attr_o == NULL); + ERROR_IF(PyStackRef_IsNull(*attr)); self_or_null[0] = PyStackRef_NULL; } } else { /* Classic, pushes one value. */ - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL); + *attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); } macro(LOAD_ATTR) = @@ -2642,12 +2641,6 @@ dummy_func( PyDictObject *dict = _PyObject_GetManagedDict(owner_o); DEOPT_IF(dict == NULL); DEOPT_IF(!LOCK_OBJECT(dict)); - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - DEOPT_IF(true); - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || diff --git a/Python/ceval.c b/Python/ceval.c index d1de4875656fd3..291e753dec0ce5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -346,13 +346,13 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (here_addr > _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES) { + if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } - return here_addr <= _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES; + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; } void @@ -448,8 +448,8 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) _tstate->c_stack_top = (uintptr_t)high; ULONG guarantee = 0; SetThreadStackGuarantee(&guarantee); - _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + PYOS_STACK_MARGIN_BYTES; - _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + PYOS_STACK_MARGIN_BYTES; + _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES; #else uintptr_t here_addr = _Py_get_machine_stack_pointer(); # if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && !defined(__NetBSD__) @@ -469,9 +469,9 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) // Thread sanitizer crashes if we use a bit more than half the stack. _tstate->c_stack_soft_limit = base + (stack_size / 2); #else - _tstate->c_stack_soft_limit = base + PYOS_STACK_MARGIN_BYTES * 2; + _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; #endif - _tstate->c_stack_hard_limit = base + PYOS_STACK_MARGIN_BYTES; + _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; assert(_tstate->c_stack_soft_limit < here_addr); assert(here_addr < _tstate->c_stack_top); return; @@ -479,7 +479,7 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) # endif _tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096); _tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE; - _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + PYOS_STACK_MARGIN_BYTES); + _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + _PyOS_STACK_MARGIN_BYTES); #endif } @@ -627,12 +627,14 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) PyObject *seen = NULL; PyObject *dummy = NULL; PyObject *values = NULL; - PyObject *get = NULL; // We use the two argument form of map.get(key, default) for two reasons: // - Atomically check for a key and get its value without error handling. // - Don't cause key creation or resizing in dict subclasses like // collections.defaultdict that define __missing__ (or similar). - int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + int meth_found = _PyObject_GetMethodStackRef(tstate, map, &_Py_ID(get), &cref.ref); + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); if (get == NULL) { goto fail; } @@ -682,12 +684,12 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) } // Success: done: - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); Py_DECREF(seen); Py_DECREF(dummy); return values; fail: - Py_XDECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); Py_XDECREF(seen); Py_XDECREF(dummy); Py_XDECREF(values); diff --git a/Python/codegen.c b/Python/codegen.c index 27fe8e1957b2fe..53388592330a5d 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4081,16 +4081,6 @@ codegen_template_str(compiler *c, expr_ty e) } else { VISIT(c, expr, value); - Py_ssize_t j; - for (j = i + 1; j < value_count; j++) { - value = asdl_seq_GET(e->v.TemplateStr.values, j); - if (value->kind == Interpolation_kind) { - break; - } - VISIT(c, expr, value); - ADDOP_INPLACE(c, loc, Add); - } - i = j - 1; stringslen++; last_was_interpolation = 0; } diff --git a/Python/critical_section.c b/Python/critical_section.c index 73857b85496316..e628ba2f6d19bc 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -130,6 +130,15 @@ PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) #endif } +#undef PyCriticalSection_BeginMutex +void +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_BeginMutex(c, m); +#endif +} + #undef PyCriticalSection_End void PyCriticalSection_End(PyCriticalSection *c) @@ -148,6 +157,15 @@ PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) #endif } +#undef PyCriticalSection2_BeginMutex +void +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_BeginMutex(c, m1, m2); +#endif +} + #undef PyCriticalSection2_End void PyCriticalSection2_End(PyCriticalSection2 *c) diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c new file mode 100644 index 00000000000000..98ee44276e53e0 --- /dev/null +++ b/Python/emscripten_syscalls.c @@ -0,0 +1,323 @@ +#include "emscripten.h" +#include "stdio.h" + +// All system calls: return nonnegative number on success, return -errno on +// failure. Negative results get stored back into errno here: +// https://github.com/emscripten-core/emscripten/blob/main/system/lib/libc/musl/src/internal/syscall_ret.c#L7 + +// If we're running in node, report the UID of the user in the native system as +// the UID of the user. Since the nodefs will report the uid correctly, if we +// don't make getuid report it correctly too we'll see some permission errors. +// Normally __syscall_getuid32 is a stub that always returns 0 but it is +// defined with weak linkage so we can override it. +EM_JS(int, __syscall_getuid32_js, (void), { + // If we're in node and we can, report the native uid + if (ENVIRONMENT_IS_NODE) { + return process.getuid(); + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_getuid32(void) { + return __syscall_getuid32_js(); +} + +EM_JS(int, __syscall_umask_js, (int mask), { + // If we're in node and we can, call native process.umask() + if (ENVIRONMENT_IS_NODE) { + try { + return process.umask(mask); + } catch(e) { + // oops... + // NodeJS docs: "In Worker threads, process.umask(mask) will throw an exception." + // umask docs: "This system call always succeeds" + return 0; + } + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_umask(int mask) { + return __syscall_umask_js(mask); +} + +#include <wasi/api.h> +#include <errno.h> +#include <fcntl.h> + +// Variant of EM_JS that does C preprocessor substitution on the body +#define EM_JS_MACROS(ret, func_name, args, body...) \ + EM_JS(ret, func_name, args, body) + +EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { + // Define FS.createAsyncInputDevice(), This is quite similar to + // FS.createDevice() defined here: + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 + // but instead of returning one byte at a time, the input() function should + // return a Uint8Array. This makes the handler code simpler, the + // `createAsyncInputDevice` simpler, and everything faster. + FS.createAsyncInputDevice = function(parent, name, input) { + parent = typeof parent == 'string' ? parent : FS.getPath(parent); + var path = PATH.join2(parent, name); + var mode = FS_getMode(true, false); + FS.createDevice.major ||= 64; + var dev = FS.makedev(FS.createDevice.major++, 0); + async function getDataBuf() { + var buf; + try { + buf = await input(); + } catch (e) { + throw new FS.ErrnoError(EIO); + } + if (!buf?.byteLength) { + throw new FS.ErrnoError(EAGAIN); + } + ops._dataBuf = buf; + } + + var ops = { + _dataBuf: new Uint8Array(0), + open(stream) { + stream.seekable = false; + }, + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + buffer = buffer.subarray(offset, offset + length); + if (!ops._dataBuf.byteLength) { + await getDataBuf(); + } + var toRead = Math.min(ops._dataBuf.byteLength, buffer.byteLength); + buffer.subarray(0, toRead).set(ops._dataBuf); + buffer = buffer.subarray(toRead); + ops._dataBuf = ops._dataBuf.subarray(toRead); + if (toRead) { + stream.node.atime = Date.now(); + } + return toRead; + }, + }; + FS.registerDevice(dev, ops); + return FS.mkdev(path, mode, dev); + }; + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + const origResolveGlobalSymbol = resolveGlobalSymbol; + if (ENVIRONMENT_IS_NODE && !Module.onExit) { + Module.onExit = (code) => process.exit(code); + } + // * wrap the main symbol with WebAssembly.promising, + // * call exit_with_live_runtime() to prevent emscripten from shutting down + // the runtime before the promise resolves, + // * call onExit / process.exit ourselves, since exit_with_live_runtime() + // prevented Emscripten from calling it normally. + resolveGlobalSymbol = function (name, direct = false) { + const orig = origResolveGlobalSymbol(name, direct); + if (name === "main") { + const main = WebAssembly.promising(orig.sym); + orig.sym = (...args) => { + (async () => { + const ret = await main(...args); + Module.onExit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + } + return orig; + }; +}) + +__attribute__((constructor)) void _emscripten_promising_main(void) { + _emscripten_promising_main_js(); +} + + +#define IOVEC_T_BUF_OFFSET 0 +#define IOVEC_T_BUF_LEN_OFFSET 4 +#define IOVEC_T_SIZE 8 +_Static_assert(offsetof(__wasi_iovec_t, buf) == IOVEC_T_BUF_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(offsetof(__wasi_iovec_t, buf_len) == IOVEC_T_BUF_LEN_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, + "Unexpected __wasi_iovec_t layout"); + +// If the stream has a readAsync handler, read to buffer defined in iovs, write +// number of bytes read to *nread, and return a promise that resolves to the +// errno. Otherwise, return null. +EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( + __wasi_fd_t fd, + const __wasi_iovec_t *iovs, + size_t iovcnt, + __wasi_size_t *nread +), { + if (!WebAssembly.promising) { + return null; + } + var stream; + try { + stream = SYSCALLS.getStreamFromFD(fd); + } catch (e) { + // If the fd was already closed or never existed, getStreamFromFD() + // raises. We'll let fd_read_orig() handle setting errno. + return null; + } + if (!stream.stream_ops.readAsync) { + // Not an async device. Fall back to __wasi_fd_read_orig(). + return null; + } + return (async () => { + // This is the same as libwasi.js fd_read() and doReadv() except we use + // readAsync and we await it. + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L331 + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L197 + try { + var ret = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[(iovs + IOVEC_T_BUF_OFFSET)/4]; + var len = HEAP32[(iovs + IOVEC_T_BUF_LEN_OFFSET)/4]; + iovs += IOVEC_T_SIZE; + var curr = await stream.stream_ops.readAsync(stream, HEAP8, ptr, len); + if (curr < 0) return -1; + ret += curr; + if (curr < len) break; // nothing more to read + } + HEAP32[nread/4] = ret; + return 0; + } catch (e) { + if (e.name !== 'ErrnoError') { + throw e; + } + return e["errno"]; + } + })(); +}; +); + +// Bind original fd_read syscall to __wasi_fd_read_orig(). +__wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("fd_read"), __warn_unused_result__)); + +// Take a promise that resolves to __wasi_errno_t and suspend until it resolves, +// get the output. +EM_JS(int, __block_for_int, (__externref_t p), { + return p; +} +if (WebAssembly.Suspending) { + __block_for_int = new WebAssembly.Suspending(__block_for_int); +} +) + +// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_int +// to get the result. +__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) { + __externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread); + if (__builtin_wasm_ref_is_null_extern(p)) { + return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); + } + return __block_for_int(p); +} + +#include <poll.h> +#define POLLFD_FD 0 +#define POLLFD_EVENTS 4 +#define POLLFD_REVENTS 6 +#define POLLFD_SIZE 8 +_Static_assert(offsetof(struct pollfd, fd) == 0, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, events) == 4, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct layout"); +_Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); + +EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { + if (!WebAssembly.promising) { + return null; + } + return (async function() { + try { + var nonzero = 0; + var promises = []; + for (var i = 0; i < nfds; i++) { + var pollfd = fds + POLLFD_SIZE * i; + var fd = HEAP32[(pollfd + POLLFD_FD)/4]; + var events = HEAP16[(pollfd + POLLFD_EVENTS)/2]; + var mask = POLLNVAL; + var stream = FS.getStream(fd); + if (stream) { + mask = POLLIN | POLLOUT; + if (stream.stream_ops.pollAsync) { + promises.push(stream.stream_ops.pollAsync(stream, timeout).then((mask) => { + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + })); + } else if (stream.stream_ops.poll) { + var mask = stream.stream_ops.poll(stream, timeout); + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + } + } + } + await Promise.all(promises); + return nonzero; + } catch(e) { + if (e?.name !== "ErrnoError") throw e; + return -e["errno"]; + } + })(); +}); + +// Bind original poll syscall to syscall_poll_orig(). +int syscall_poll_orig(intptr_t fds, int nfds, int timeout) + __attribute__((__import_module__("env"), + __import_name__("__syscall_poll"), __warn_unused_result__)); + +int __syscall_poll(intptr_t fds, int nfds, int timeout) { + __externref_t p = __maybe_poll_async(fds, nfds, timeout); + if (__builtin_wasm_ref_is_null_extern(p)) { + return syscall_poll_orig(fds, nfds, timeout); + } + return __block_for_int(p); +} + +#include <sys/ioctl.h> + +int syscall_ioctl_orig(int fd, int request, void* varargs) + __attribute__((__import_module__("env"), + __import_name__("__syscall_ioctl"), __warn_unused_result__)); + +int __syscall_ioctl(int fd, int request, void* varargs) { + if (request == FIOCLEX || request == FIONCLEX) { + return 0; + } + if (request == FIONBIO) { + // Implement FIONBIO via fcntl. + // TODO: Upstream this. + int flags = fcntl(fd, F_GETFL, 0); + int nonblock = **((int**)varargs); + if (flags < 0) { + return -errno; + } + if (nonblock) { + flags |= O_NONBLOCK; + } else { + flags &= (~O_NONBLOCK); + } + int res = fcntl(fd, F_SETFL, flags); + if (res < 0) { + return -errno; + } + return res; + } + return syscall_ioctl_orig(fd, request, varargs); +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 46fc164a5b3bc2..e152865e4ec9e8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3301,20 +3301,20 @@ case _LOAD_ATTR: { _PyStackRef owner; - _PyStackRef attr; + _PyStackRef *attr; _PyStackRef *self_or_null; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; + *attr = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { - assert(attr_o != NULL); + assert(!PyStackRef_IsNull(*attr)); self_or_null[0] = owner; } else { @@ -3323,7 +3323,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { + if (PyStackRef_IsNull(*attr)) { JUMP_TO_ERROR(); } self_or_null[0] = PyStackRef_NULL; @@ -3332,7 +3332,7 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3342,10 +3342,9 @@ if (attr_o == NULL) { JUMP_TO_ERROR(); } + *attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); break; @@ -3705,15 +3704,6 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 2adc8c84d83974..1cb6f03169e3b5 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1892,6 +1892,10 @@ eval_const_unaryop(PyObject *operand, int opcode, int oparg) result = PyNumber_Negative(operand); break; case UNARY_INVERT: + // XXX: This should be removed once the ~bool depreciation expires. + if (PyBool_Check(operand)) { + return NULL; + } result = PyNumber_Invert(operand); break; case UNARY_NOT: { diff --git a/Python/gc.c b/Python/gc.c index 7b0e6d6e803504..2e697faac032b5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1,6 +1,6 @@ // This implements the reference cycle garbage collector. // The Python module interface to the collector is in gcmodule.c. -// See https://devguide.python.org/internals/garbage-collector/ +// See InternalDocs/garbage_collector.md for more infromation. #include "Python.h" #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() @@ -870,7 +870,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) * no object in `unreachable` is weakly referenced anymore. */ static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) { PyGC_Head *gc; PyObject *op; /* generally FROM_GC(gc) */ @@ -879,7 +879,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) PyGC_Head *next; int num_freed = 0; - gc_list_init(&wrcb_to_call); + if (allow_callbacks) { + gc_list_init(&wrcb_to_call); + } /* Clear all weakrefs to the objects in unreachable. If such a weakref * also has a callback, move it into `wrcb_to_call` if the callback @@ -935,6 +937,11 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); _PyWeakref_ClearRef(wr); _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + + if (!allow_callbacks) { + continue; + } + if (wr->wr_callback == NULL) { /* no callback */ continue; @@ -987,6 +994,10 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) } } + if (!allow_callbacks) { + return 0; + } + /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ @@ -1320,15 +1331,6 @@ gc_collect_young(PyThreadState *tstate, PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; untrack_tuples(young); GC_STAT_ADD(0, collections, 1); -#ifdef Py_STATS - { - Py_ssize_t count = 0; - PyGC_Head *gc; - for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { - count++; - } - } -#endif PyGC_Head survivors; gc_list_init(&survivors); @@ -1394,7 +1396,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc_list_move(gc, &gcstate->permanent_generation.head); gc = next; continue; } @@ -1737,7 +1739,7 @@ gc_collect_region(PyThreadState *tstate, } /* Clear weakrefs and invoke callbacks as necessary. */ - stats->collected += handle_weakrefs(&unreachable, to); + stats->collected += handle_weakrefs(&unreachable, to, true); gc_list_validate_space(to, gcstate->visited_space); validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); @@ -1751,6 +1753,14 @@ gc_collect_region(PyThreadState *tstate, gc_list_init(&final_unreachable); handle_resurrected_objects(&unreachable, &final_unreachable, to); + /* Clear weakrefs to objects in the unreachable set. No Python-level + * code must be allowed to access those unreachable objects. During + * delete_garbage(), finalizers outside the unreachable set might run + * and create new weakrefs. If those weakrefs were not cleared, they + * could reveal unreachable objects. Callbacks are not executed. + */ + handle_weakrefs(&final_unreachable, NULL, false); + /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. @@ -1763,7 +1773,7 @@ gc_collect_region(PyThreadState *tstate, Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } stats->uncollectable = n; diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 5aaa68c5b51f95..0b0ddf227e4952 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1073,6 +1073,14 @@ validate_refcounts(const mi_heap_t *heap, const mi_heap_area_t *area, return true; } + // This assert mirrors the one in Python/gc.c:update_refs(). There must be + // no tracked objects with a reference count of 0 when the cyclic + // collector starts. If there is, then the collector will double dealloc + // the object. The likely cause for hitting this is a faulty .tp_dealloc. + // Also see the comment in `update_refs()`. + _PyObject_ASSERT_WITH_MSG(op, Py_REFCNT(op) > 0, + "tracked objects must have a reference count > 0"); + _PyObject_ASSERT_WITH_MSG(op, !gc_is_unreachable(op), "object should not be marked as unreachable yet"); @@ -1422,13 +1430,6 @@ static int deduce_unreachable_heap(PyInterpreterState *interp, struct collection_state *state) { - -#ifdef GC_DEBUG - // Check that all objects are marked as unreachable and that the computed - // reference count difference (stored in `ob_tid`) is non-negative. - gc_visit_heaps(interp, &validate_refcounts, &state->base); -#endif - // Identify objects that are directly reachable from outside the GC heap // by computing the difference between the refcount and the number of // incoming references. @@ -1492,9 +1493,9 @@ move_legacy_finalizer_reachable(struct collection_state *state) } // Clear all weakrefs to unreachable objects. Weakrefs with callbacks are -// enqueued in `wrcb_to_call`, but not invoked yet. +// optionally enqueued in `wrcb_to_call`, but not invoked yet. static void -clear_weakrefs(struct collection_state *state) +clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { @@ -1526,6 +1527,10 @@ clear_weakrefs(struct collection_state *state) _PyWeakref_ClearRef(wr); _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + if (!enqueue_callbacks) { + continue; + } + // We do not invoke callbacks for weakrefs that are themselves // unreachable. This is partly for historical reasons: weakrefs // predate safe object finalization, and a weakref that is itself @@ -2154,6 +2159,13 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, state->gcstate->old[i-1].count = 0; } +#ifdef GC_DEBUG + // Before we start, check that the heap is in a good condition. There must + // be no objects with a zero reference count. And `ob_tid` must only have a + // thread if the refcount is unmerged. + gc_visit_heaps(interp, &validate_refcounts, &state->base); +#endif + _Py_FOR_EACH_TSTATE_BEGIN(interp, p) { _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p; @@ -2211,7 +2223,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, interp->gc.long_lived_total = state->long_lived_total; // Clear weakrefs and enqueue callbacks (but do not call them). - clear_weakrefs(state); + clear_weakrefs(state, true); _PyEval_StartTheWorld(interp); // Deallocate any object from the refcount merge step @@ -2222,11 +2234,19 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, call_weakref_callbacks(state); finalize_garbage(state); - // Handle any objects that may have resurrected after the finalization. _PyEval_StopTheWorld(interp); + // Handle any objects that may have resurrected after the finalization. err = handle_resurrected_objects(state); // Clear free lists in all threads _PyGC_ClearAllFreeLists(interp); + if (err == 0) { + // Clear weakrefs to objects in the unreachable set. No Python-level + // code must be allowed to access those unreachable objects. During + // delete_garbage(), finalizers outside the unreachable set might + // run and create new weakrefs. If those weakrefs were not cleared, + // they could reveal unreachable objects. + clear_weakrefs(state, false); + } _PyEval_StartTheWorld(interp); if (err < 0) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8f7932f0033c6f..aa1eb373b7ba4b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7941,7 +7941,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; - _PyStackRef attr; + _PyStackRef *attr; _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { @@ -7964,16 +7964,16 @@ /* Skip 8 cache entries */ // _LOAD_ATTR { + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; + *attr = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { - assert(attr_o != NULL); + assert(!PyStackRef_IsNull(*attr)); self_or_null[0] = owner; } else { @@ -7982,7 +7982,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { + if (PyStackRef_IsNull(*attr)) { JUMP_TO_LABEL(error); } self_or_null[0] = PyStackRef_NULL; @@ -7991,7 +7991,7 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8001,11 +8001,10 @@ if (attr_o == NULL) { JUMP_TO_LABEL(error); } + *attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -11065,16 +11064,6 @@ assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } - #ifdef Py_GIL_DISABLED - if (dict != _PyObject_GetManagedDict(owner_o)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); - } - } - #endif assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || diff --git a/Python/getargs.c b/Python/getargs.c index 0cf596285cc7b5..c119ca5c35398b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,8 +1,6 @@ /* New getargs implementation */ -#include <stdbool.h> - #define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() @@ -468,12 +466,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; - bool nullable = false; int istuple = PyTuple_Check(arg); int mustbetuple = istuple; - assert(*format == '('); - format++; for (;;) { int c = *format++; if (c == '(') { @@ -482,12 +477,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, level++; } else if (c == ')') { - if (level == 0) { - if (*format == '?') { - nullable = true; - } + if (level == 0) break; - } level--; } else if (c == ':' || c == ';' || c == '\0') @@ -524,13 +515,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - if (arg == Py_None && nullable) { - const char *msg = skipitem(p_format, p_va, flags); - if (msg != NULL) { - levels[0] = 0; - } - return msg; - } if (istuple) { /* fallthrough */ } @@ -539,10 +523,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item tuple%s, not %.50s", - n, - nullable ? " or None" : "", - arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); + "must be %d-item tuple, not %.50s", + n, + arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } else { @@ -579,7 +562,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return msgbuf; } - format = *p_format + 1; + format = *p_format; for (i = 0; i < n; i++) { const char *msg; PyObject *item = PyTuple_GET_ITEM(arg, i); @@ -594,10 +577,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - format++; - if (*format == '?') { - format++; - } *p_format = format; if (!istuple) { Py_DECREF(arg); @@ -616,8 +595,11 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; if (*format == '(' /* ')' */) { + format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, bufsize, freelist); + if (msg == NULL) + format++; } else { msg = convertsimple(arg, &format, p_va, flags, @@ -647,7 +629,7 @@ _PyArg_BadArgument(const char *fname, const char *displayname, } static const char * -converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); assert(arg != NULL); @@ -657,23 +639,20 @@ converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, siz } else { PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s", expected, - nullable ? " or None" : "", + "must be %.50s, not %.50s", expected, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); } return msgbuf; } static const char * -convertcharerr(bool nullable, const char *expected, const char *what, Py_ssize_t size, +convertcharerr(const char *expected, const char *what, Py_ssize_t size, char *msgbuf, size_t bufsize) { assert(expected != NULL); PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s of length %zd", - expected, - nullable ? " or None" : "", - what, size); + "must be %.50s, not %.50s of length %zd", + expected, what, size); return msgbuf; } @@ -693,26 +672,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { #define RETURN_ERR_OCCURRED return msgbuf -#define HANDLE_NULLABLE \ - if (*format == '?') { \ - format++; \ - if (arg == Py_None) { \ - break; \ - } \ - nullable = true; \ - } - const char *format = *p_format; char c = *format++; const char *sarg; - bool nullable = false; switch (c) { case 'b': { /* unsigned byte -- very short int */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -726,6 +694,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "unsigned byte integer is greater than maximum"); RETURN_ERR_OCCURRED; } + else *p = (unsigned char) ival; break; } @@ -733,18 +702,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned char), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned char) ival; + } + if ((size_t)bytes > sizeof(unsigned char)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'h': {/* signed short int */ short *p = va_arg(*p_va, short *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -766,18 +742,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ unsigned short *p = va_arg(*p_va, unsigned short *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned short), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned short) ival; + } + if ((size_t)bytes > sizeof(unsigned short)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -799,12 +782,20 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'I': { /* int sized bitfield, both signed and unsigned allowed */ unsigned int *p = va_arg(*p_va, unsigned int *); - HANDLE_NULLABLE; - unsigned long ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)-1 && PyErr_Occurred()) + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned int), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; - else - *p = (unsigned int) ival; + } + if ((size_t)bytes > sizeof(unsigned int)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } @@ -812,7 +803,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - HANDLE_NULLABLE; Py_ssize_t ival = -1; iobj = _PyNumber_Index(arg); if (iobj != NULL) { @@ -826,7 +816,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'l': {/* long int */ long *p = va_arg(*p_va, long *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -837,22 +826,28 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); - HANDLE_NULLABLE; - unsigned long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } - ival = PyLong_AsUnsignedLongMask(arg); - if (ival == (unsigned long)(long)-1 && PyErr_Occurred()) { + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; } - *p = ival; + if ((size_t)bytes > sizeof(unsigned long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'L': {/* long long */ long long *p = va_arg( *p_va, long long * ); - HANDLE_NULLABLE; long long ival = PyLong_AsLongLong(arg); if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -863,22 +858,28 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'K': { /* long long sized bitfield */ unsigned long long *p = va_arg(*p_va, unsigned long long *); - HANDLE_NULLABLE; - unsigned long long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } - ival = PyLong_AsUnsignedLongLongMask(arg); - if (ival == (unsigned long long)(long long)-1 && PyErr_Occurred()) { + Py_ssize_t bytes = PyLong_AsNativeBytes(arg, p, sizeof(unsigned long long), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (bytes < 0) { RETURN_ERR_OCCURRED; } - *p = ival; + if ((size_t)bytes > sizeof(unsigned long long)) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + { + RETURN_ERR_OCCURRED; + } + } break; } case 'f': {/* float */ float *p = va_arg(*p_va, float *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -889,7 +890,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'd': {/* double */ double *p = va_arg(*p_va, double *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -900,7 +900,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); - HANDLE_NULLABLE; Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) @@ -912,10 +911,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) { if (PyBytes_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytes object", PyBytes_GET_SIZE(arg), msgbuf, bufsize); } @@ -923,28 +921,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (PyByteArray_Check(arg)) { if (PyByteArray_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytearray object", PyByteArray_GET_SIZE(arg), msgbuf, bufsize); } *p = PyByteArray_AS_STRING(arg)[0]; } else - return converterr(nullable, "a byte string of length 1", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int kind; const void *data; if (!PyUnicode_Check(arg)) - return converterr(nullable, "a unicode character", arg, msgbuf, bufsize); + return converterr("a unicode character", arg, msgbuf, bufsize); if (PyUnicode_GET_LENGTH(arg) != 1) { - return convertcharerr(nullable, "a unicode character", + return convertcharerr("a unicode character", "a string", PyUnicode_GET_LENGTH(arg), msgbuf, bufsize); } @@ -957,7 +954,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'p': {/* boolean *p*redicate */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int val = PyObject_IsTrue(arg); if (val > 0) *p = 1; @@ -976,31 +972,24 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count; if (*format == '*') { - format++; - HANDLE_NULLABLE; if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); + format++; if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } - else if (*format == '#') { + count = convertbuffer(arg, (const void **)p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); *psize = count; - } - else { - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + format++; + } else { if (strlen(*p) != (size_t)count) { PyErr_SetString(PyExc_ValueError, "embedded null byte"); RETURN_ERR_OCCURRED; @@ -1016,35 +1005,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, (void *)sarg, len, 1, 0); } else { /* any bytes-like object */ const char *buf; if (getbuffer(arg, p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); } if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } + format++; } else if (*format == '#') { /* a string or read-only bytes-like object */ /* "s#" or "z#" */ const void **p = (const void **)va_arg(*p_va, const char **); Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) { *p = NULL; *psize = 0; @@ -1053,7 +1039,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); *p = sarg; *psize = len; @@ -1063,22 +1049,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); *psize = count; } + format++; } else { /* "s" or "z" */ const char **p = va_arg(*p_va, const char **); Py_ssize_t len; sarg = NULL; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); if (strlen(sarg) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "embedded null character"); @@ -1087,7 +1073,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = sarg; } else - return converterr(c == 'z' || nullable, "str", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); } break; @@ -1116,46 +1102,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, recode_strings = 0; else return converterr( - nullable, "(unknown parser marker combination)", + "(unknown parser marker combination)", arg, msgbuf, bufsize); buffer = (char **)va_arg(*p_va, char **); format++; if (buffer == NULL) - return converterr(nullable, "(buffer is NULL)", + return converterr("(buffer is NULL)", arg, msgbuf, bufsize); - Py_ssize_t *psize = NULL; - if (*format == '#') { - /* Using buffer length parameter '#': - - - if *buffer is NULL, a new buffer of the - needed size is allocated and the data - copied into it; *buffer is updated to point - to the new buffer; the caller is - responsible for PyMem_Free()ing it after - usage - - - if *buffer is not NULL, the data is - copied to *buffer; *buffer_len has to be - set to the size of the buffer on input; - buffer overflow is signalled with an error; - buffer has to provide enough room for the - encoded string plus the trailing 0-byte - - - in both cases, *buffer_len is updated to - the size of the buffer /excluding/ the - trailing 0-byte - - */ - psize = va_arg(*p_va, Py_ssize_t*); - - format++; - if (psize == NULL) { - return converterr( - nullable, "(buffer_len is NULL)", - arg, msgbuf, bufsize); - } - } - HANDLE_NULLABLE; /* Encode object */ if (!recode_strings && @@ -1176,7 +1129,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, encoding, NULL); if (s == NULL) - return converterr(nullable, "(encoding failed)", + return converterr("(encoding failed)", arg, msgbuf, bufsize); assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); @@ -1186,15 +1139,42 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { return converterr( - nullable, - recode_strings ? "str" - : nullable ? "str, bytes, bytearray" - : "str, bytes or bytearray", + recode_strings ? "str" : "str, bytes or bytearray", arg, msgbuf, bufsize); } /* Write output; output is guaranteed to be 0-terminated */ - if (psize != NULL) { + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer of the + needed size is allocated and the data + copied into it; *buffer is updated to point + to the new buffer; the caller is + responsible for PyMem_Free()ing it after + usage + + - if *buffer is not NULL, the data is + copied to *buffer; *buffer_len has to be + set to the size of the buffer on input; + buffer overflow is signalled with an error; + buffer has to provide enough room for the + encoded string plus the trailing 0-byte + + - in both cases, *buffer_len is updated to + the size of the buffer /excluding/ the + trailing 0-byte + + */ + Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); + + format++; + if (psize == NULL) { + Py_DECREF(s); + return converterr( + "(buffer_len is NULL)", + arg, msgbuf, bufsize); + } if (*buffer == NULL) { *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { @@ -1205,7 +1185,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } } else { @@ -1239,7 +1219,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( - nullable, "encoded string without null bytes", + "encoded string without null bytes", arg, msgbuf, bufsize); } *buffer = PyMem_NEW(char, size + 1); @@ -1250,7 +1230,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } memcpy(*buffer, ptr, size+1); @@ -1261,32 +1241,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'S': { /* PyBytes object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) *p = arg; else - return converterr(nullable, "bytes", arg, msgbuf, bufsize); + return converterr("bytes", arg, msgbuf, bufsize); break; } case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyByteArray_Check(arg)) *p = arg; else - return converterr(nullable, "bytearray", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyUnicode_Check(arg)) { *p = arg; } else - return converterr(nullable, "str", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } @@ -1297,11 +1274,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, type = va_arg(*p_va, PyTypeObject*); p = va_arg(*p_va, PyObject **); format++; - HANDLE_NULLABLE; if (PyType_IsSubtype(Py_TYPE(arg), type)) *p = arg; else - return converterr(nullable, type->tp_name, arg, msgbuf, bufsize); + return converterr(type->tp_name, arg, msgbuf, bufsize); } else if (*format == '&') { @@ -1310,18 +1286,16 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void *addr = va_arg(*p_va, void *); int res; format++; - HANDLE_NULLABLE; if (! (res = (*convert)(arg, addr))) - return converterr(nullable, "(unspecified)", + return converterr("(unspecified)", arg, msgbuf, bufsize); if (res == Py_CLEANUP_SUPPORTED && addcleanup(addr, freelist, convert) == -1) - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; *p = arg; } break; @@ -1333,30 +1307,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format != '*') return converterr( - nullable, "(invalid use of 'w' format character)", + "(invalid use of 'w' format character)", arg, msgbuf, bufsize); format++; - HANDLE_NULLABLE; /* Caller is interested in Py_buffer, and the object supports it directly. The request implicitly asks for PyBUF_SIMPLE, so the result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - return converterr(nullable, "read-write bytes-like object", + return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } default: - return converterr(nullable, "(impossible<bad format char>)", arg, msgbuf, bufsize); + return converterr("(impossible<bad format char>)", arg, msgbuf, bufsize); } @@ -2751,9 +2724,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) return "impossible<bad format char>"; } - if (*format == '?') { - format++; - } *p_format = format; return NULL; diff --git a/Python/initconfig.c b/Python/initconfig.c index 71d7cfed5c44c1..cc0db19d416058 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -312,7 +312,7 @@ The following implementation-specific options are available:\n\ "-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" #endif "\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\ +-X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ @@ -459,7 +459,7 @@ static const char usage_envvars[] = /* --- Global configuration variables ----------------------------- */ -/* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change +/* UTF-8 mode (PEP 540): if equal to 1, use the UTF-8 encoding, and change stdin and stdout error handler to "surrogateescape". */ int Py_UTF8Mode = 0; int Py_DebugFlag = 0; /* Needed by parser.c */ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 13bdd041becd69..f9913cff402ed1 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1190,9 +1190,10 @@ call_instrumentation_vector( break; } else { - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_tools(code, offset, event, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } } @@ -1381,9 +1382,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_line_tools(code, i, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } while (tools); Py_DECREF(line_obj); @@ -1438,9 +1440,10 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_per_instruction_tools(code, offset, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } Py_DECREF(offset_obj); @@ -1530,7 +1533,7 @@ initialize_lines(PyCodeObject *code, int bytes_per_entry) case END_FOR: case END_SEND: case RESUME: - case POP_ITER: + case POP_ITER: /* END_FOR cannot start a line, as it is skipped by FOR_ITER * END_SEND cannot start a line, as it is skipped by SEND * RESUME and POP_ITER must not be instrumented with INSTRUMENTED_LINE */ @@ -2558,18 +2561,22 @@ PyObject *_Py_CreateMonitoringObject(void) err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + assert(val != NULL); /* Can't return NULL because the int is small. */ err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "PROFILER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); Py_DECREF(val); if (err) goto error; @@ -2991,9 +2998,10 @@ branch_handler_vectorcall( // Orphaned NOT_TAKEN -- Jump removed by the compiler return res; } - LOCK_CODE(code); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); remove_tools(code, offset, other_event, 1 << self->tool_id); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } return res; } diff --git a/Python/jit.c b/Python/jit.c index e232cc1f7d9250..01bc0076497c6d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -431,8 +431,10 @@ void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *s #if defined(__aarch64__) || defined(_M_ARM64) #define TRAMPOLINE_SIZE 16 + #define DATA_ALIGN 8 #else #define TRAMPOLINE_SIZE 0 + #define DATA_ALIGN 1 #endif // Generate and patch AArch64 trampolines. The symbols to jump to are stored @@ -522,8 +524,9 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); - size_t padding = page_size - ((code_size + state.trampolines.size + data_size) & (page_size - 1)); - size_t total_size = code_size + state.trampolines.size + data_size + padding; + size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); + size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size) & (page_size - 1)); + size_t total_size = code_size + state.trampolines.size + code_padding + data_size + padding; unsigned char *memory = jit_alloc(total_size); if (memory == NULL) { return -1; @@ -545,7 +548,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Loop again to emit the code: unsigned char *code = memory; state.trampolines.mem = memory + code_size; - unsigned char *data = memory + code_size + state.trampolines.size; + unsigned char *data = memory + code_size + state.trampolines.size + code_padding; // Compile the shim, which handles converting between the native // calling convention and the calling convention used by jitted code // (which may be different for efficiency reasons). @@ -567,7 +570,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz code += group->code_size; data += group->data_size; assert(code == memory + code_size); - assert(data == memory + code_size + state.trampolines.size + data_size); + assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); #ifdef MAP_JIT pthread_jit_write_protect_np(1); #endif diff --git a/Python/lock.c b/Python/lock.c index ea6ac00bfeccb4..a49d587a1686d9 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -95,6 +95,18 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) if (timeout == 0) { return PY_LOCK_FAILURE; } + if ((flags & _PY_LOCK_PYTHONLOCK) && Py_IsFinalizing()) { + // At this phase of runtime shutdown, only the finalization thread + // can have attached thread state; others hang if they try + // attaching. And since operations on this lock requires attached + // thread state (_PY_LOCK_PYTHONLOCK), the finalization thread is + // running this code, and no other thread can unlock. + // Raise rather than hang. (_PY_LOCK_PYTHONLOCK allows raising + // exceptons.) + PyErr_SetString(PyExc_PythonFinalizationError, + "cannot acquire lock at interpreter finalization"); + return PY_LOCK_FAILURE; + } uint8_t newv = v; if (!(v & _Py_HAS_PARKED)) { @@ -622,3 +634,11 @@ PyMutex_Unlock(PyMutex *m) Py_FatalError("unlocking mutex that is not locked"); } } + + +#undef PyMutex_IsLocked +int +PyMutex_IsLocked(PyMutex *m) +{ + return _PyMutex_IsLocked(m); +} diff --git a/Python/marshal.c b/Python/marshal.c index afbef6ee6796d9..15dd25d6268df4 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1656,6 +1656,9 @@ r_object(RFILE *p) case TYPE_SLICE: { Py_ssize_t idx = r_ref_reserve(flag, p); + if (idx < 0) { + break; + } PyObject *stop = NULL; PyObject *step = NULL; PyObject *start = r_object(p); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3182e8b3b70144..aeff76affd8ace 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -590,9 +590,9 @@ dummy_func(void) { } } - op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { (void)owner; - attr = sym_new_not_null(ctx); + *attr = sym_new_not_null(ctx); if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8d30df3aa7d429..41402200c1683e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1414,16 +1414,16 @@ case _LOAD_ATTR: { JitOptRef owner; - JitOptRef attr; + JitOptRef *attr; JitOptRef *self_or_null; owner = stack_pointer[-1]; + attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - attr = sym_new_not_null(ctx); + *attr = sym_new_not_null(ctx); if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } - stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); break; @@ -2723,9 +2723,6 @@ PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -2733,6 +2730,9 @@ res_stackref = PyStackRef_FromPyObjectSteal(res_o); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); break; } bool lhs_int = sym_matches_type(lhs, &PyLong_Type); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 2ca18c23593547..8732be973616d4 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -66,7 +66,9 @@ #ifdef PY_HAVE_PERF_TRAMPOLINE /* Standard library includes for perf jitdump implementation */ -#include <elf.h> // ELF architecture constants +#if defined(__linux__) +# include <elf.h> // ELF architecture constants +#endif #include <fcntl.h> // File control operations #include <stdio.h> // Standard I/O operations #include <stdlib.h> // Standard library functions @@ -74,7 +76,9 @@ #include <sys/types.h> // System data types #include <unistd.h> // System calls (sysconf, getpid) #include <sys/time.h> // Time functions (gettimeofday) -#include <sys/syscall.h> // System call interface +#if defined(__linux__) +# include <sys/syscall.h> // System call interface +#endif // ============================================================================= // CONSTANTS AND CONFIGURATION @@ -97,10 +101,25 @@ * /tmp/jitted-PID-0.so: [headers][.text][unwind_info][padding] * /tmp/jitted-PID-1.so: [headers][.text][unwind_info][padding] * - * The padding size (0x100) is chosen to accommodate typical unwind info sizes - * while maintaining 16-byte alignment requirements. + * The padding size is now calculated automatically during initialization + * based on the actual unwind information requirements. */ -#define PERF_JIT_CODE_PADDING 0x100 + + +/* These constants are defined inside <elf.h>, which we can't use outside of linux. */ +#if !defined(__linux__) +# if defined(__i386__) || defined(_M_IX86) +# define EM_386 3 +# elif defined(__arm__) || defined(_M_ARM) +# define EM_ARM 40 +# elif defined(__x86_64__) || defined(_M_X64) +# define EM_X86_64 62 +# elif defined(__aarch64__) +# define EM_AARCH64 183 +# elif defined(__riscv) +# define EM_RISCV 243 +# endif +#endif /* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api @@ -195,7 +214,11 @@ struct BaseEvent { typedef struct { struct BaseEvent base; // Common event header uint32_t process_id; // Process ID where code was generated +#if defined(__APPLE__) + uint64_t thread_id; // Thread ID where code was generated +#else uint32_t thread_id; // Thread ID where code was generated +#endif uint64_t vma; // Virtual memory address where code is loaded uint64_t code_address; // Address of the actual machine code uint64_t code_size; // Size of the machine code in bytes @@ -401,10 +424,12 @@ enum { DWRF_CFA_nop = 0x0, // No operation DWRF_CFA_offset_extended = 0x5, // Extended offset instruction DWRF_CFA_def_cfa = 0xc, // Define CFA rule + DWRF_CFA_def_cfa_register = 0xd, // Define CFA register DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter - DWRF_CFA_offset = 0x80 // Simple offset instruction + DWRF_CFA_offset = 0x80, // Simple offset instruction + DWRF_CFA_restore = 0xc0 // Restore register }; /* DWARF Exception Handling pointer encodings */ @@ -519,6 +544,7 @@ typedef struct ELFObjectContext { uint8_t* p; // Current write position in buffer uint8_t* startp; // Start of buffer (for offset calculations) uint8_t* eh_frame_p; // Start of EH frame data (for relative offsets) + uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) uint32_t code_size; // Size of the code being described } ELFObjectContext; @@ -643,6 +669,8 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { // DWARF EH FRAME GENERATION // ============================================================================= +static void elf_init_ehframe(ELFObjectContext* ctx); + /* * Initialize DWARF .eh_frame section for a code region * @@ -657,6 +685,23 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { * Args: * ctx: ELF object context containing code size and buffer pointers */ +static size_t calculate_eh_frame_size(void) { + /* Calculate the EH frame size for the trampoline function */ + extern void *_Py_trampoline_func_start; + extern void *_Py_trampoline_func_end; + + size_t code_size = (char*)&_Py_trampoline_func_end - (char*)&_Py_trampoline_func_start; + + ELFObjectContext ctx; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; + + elf_init_ehframe(&ctx); + return ctx.p - ctx.startp; +} + static void elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; uint8_t* framep = p; // Remember start of frame data @@ -784,7 +829,7 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * * DWRF_SECTION(FDE, * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) - * DWRF_U32(-0x30); // Initial PC-relative location of the code + * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) * DWRF_U32(ctx->code_size); // Code range covered by this FDE * DWRF_U8(0); // Augmentation data length (none) * @@ -830,19 +875,31 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { DWRF_U32(0); // CIE ID (0 indicates this is a CIE) DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) - DWRF_UV(1); // Code alignment factor +#ifdef __x86_64__ + DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) +#endif DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) DWRF_U8(DWRF_REG_RA); // Return address register number DWRF_UV(1); // Augmentation data length DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // FDE pointer encoding /* Initial CFI instructions - describe default calling convention */ +#ifdef __x86_64__ + /* x86_64 initial CFI state */ DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) DWRF_UV(DWRF_REG_SP); // CFA = SP register DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved DWRF_UV(1); // At offset 1 from CFA - +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) + // No initial register saves in AArch64 CIE +#endif DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary ) @@ -853,11 +910,15 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * * The FDE describes unwinding information specific to this function. * It references the CIE and provides function-specific CFI instructions. + * + * The PC-relative offset is calculated after the entire EH frame is built + * to ensure accurate positioning relative to the synthesized DSO layout. */ DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) - DWRF_U32(-0x30); // Machine code offset relative to .text - DWRF_U32(ctx->code_size); // Address range covered by this FDE (code lenght) + ctx->fde_p = p; // Remember where PC offset field is located for later calculation + DWRF_U32(0); // Placeholder for PC-relative offset (calculated at end of elf_init_ehframe) + DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) DWRF_U8(0); // Augmentation data length (none) /* @@ -868,32 +929,36 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * conventions and register usage patterns. */ #ifdef __x86_64__ - /* x86_64 calling convention unwinding rules */ + /* x86_64 calling convention unwinding rules with frame pointer */ # if defined(__CET__) && (__CET__ & 1) - DWRF_U8(DWRF_CFA_advance_loc | 8); // Advance location by 8 bytes when CET protection is enabled -# else - DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance location by 4 bytes + DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) # endif - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) + DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 DWRF_UV(16); // New offset: SP + 16 - DWRF_U8(DWRF_CFA_advance_loc | 6); // Advance location by 6 bytes - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset + DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 + DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 + DWRF_UV(DWRF_REG_BP); // Use base pointer register + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 + DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 + DWRF_UV(DWRF_REG_SP); // Use stack pointer register DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 instruction (stp x29, x30) - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset - DWRF_UV(16); // CFA = SP + 16 (stack pointer after push) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Frame pointer (x29) saved - DWRF_UV(2); // At offset 2 from CFA (2 * 8 = 16 bytes) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Link register (x30) saved - DWRF_UV(1); // At offset 1 from CFA (1 * 8 = 8 bytes) - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (mov x16, x3; mov x29, sp; ldp...) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore frame pointer (x29) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore link register (x30) - DWRF_U8(DWRF_CFA_def_cfa_offset); // Final CFA adjustment - DWRF_UV(0); // CFA = SP + 0 (stack restored) - + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + DWRF_UV(16); // Stack pointer moved by 16 bytes + DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved + DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) + DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved + DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) + DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) + DWRF_UV(0); // Back to original stack position #else # error "Unsupported target architecture" #endif @@ -902,6 +967,58 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { ) ctx->p = p; // Update context pointer to end of generated data + + /* Calculate and update the PC-relative offset in the FDE + * + * When perf processes the jitdump, it creates a synthesized DSO with this layout: + * + * Synthesized DSO Memory Layout: + * โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” < code_start + * โ”‚ Code Section โ”‚ + * โ”‚ (round_up(code_size, 8) bytes) โ”‚ + * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค < start of EH frame data + * โ”‚ EH Frame Data โ”‚ + * โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + * โ”‚ โ”‚ CIE data โ”‚ โ”‚ + * โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + * โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + * โ”‚ โ”‚ FDE Header: โ”‚ โ”‚ + * โ”‚ โ”‚ - CIE offset (4 bytes) โ”‚ โ”‚ + * โ”‚ โ”‚ - PC offset (4 bytes) <โ”€ fde_offset_in_frame โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”ผโ”€> points to code_start + * โ”‚ โ”‚ - address range (4 bytes) โ”‚ โ”‚ (this specific field) + * โ”‚ โ”‚ CFI Instructions... โ”‚ โ”‚ + * โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + * โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค < reference_point + * โ”‚ EhFrameHeader โ”‚ + * โ”‚ (navigation metadata) โ”‚ + * โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + * + * The PC offset field in the FDE must contain the distance from itself to code_start: + * + * distance = code_start - fde_pc_field + * + * Where: + * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame + * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) + * + * Therefore: + * distance = code_start_location - fde_pc_field_location + * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) + * = -rounded_code_size - fde_offset_in_frame + * = -(round_up(code_size, 8) + fde_offset_in_frame) + * + * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field, + * + */ + if (ctx->fde_p != NULL) { + int32_t fde_offset_in_frame = (ctx->fde_p - ctx->startp); + int32_t rounded_code_size = round_up(ctx->code_size, 8); + int32_t pc_relative_offset = -(rounded_code_size + fde_offset_in_frame); + + + // Update the PC-relative offset in the FDE + *(int32_t*)ctx->fde_p = pc_relative_offset; + } } // ============================================================================= @@ -942,6 +1059,10 @@ static void* perf_map_jit_init(void) { return NULL; // Failed to get page size } +#if defined(__APPLE__) + // On macOS, samply uses a preload to find jitdumps and this mmap can be slow. + perf_jit_map_state.mapped_buffer = NULL; +#else /* * Map the first page of the jitdump file * @@ -964,6 +1085,7 @@ static void* perf_map_jit_init(void) { close(fd); return NULL; // Memory mapping failed } +#endif perf_jit_map_state.mapped_size = page_size; @@ -1002,8 +1124,11 @@ static void* perf_map_jit_init(void) { /* Initialize code ID counter */ perf_jit_map_state.code_id = 0; - /* Configure trampoline API with padding information */ - trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + /* Calculate padding size based on actual unwind info requirements */ + size_t eh_frame_size = calculate_eh_frame_size(); + size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + trampoline_api.code_padding = round_up(unwind_data_size, 16); + trampoline_api.code_alignment = 32; return &perf_jit_map_state; } @@ -1092,6 +1217,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) ctx.code_size = code_size; ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; // Initialize to NULL, will be set when FDE is written /* Generate EH frame (Exception Handling frame) data */ elf_init_ehframe(&ctx); @@ -1110,7 +1236,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; /* Verify we don't exceed our padding budget */ - assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding); ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); ev2.mapped_size = round_up(ev2.unwind_data_size, 16); // 16-byte alignment @@ -1166,7 +1292,11 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); +#if defined(__APPLE__) + pthread_threadid_np(NULL, &ev.thread_id); +#else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call +#endif ev.vma = base; // Virtual memory address ev.code_address = base; // Same as VMA for our use case ev.code_size = size; @@ -1262,4 +1392,4 @@ _PyPerf_Callbacks _Py_perfmap_jit_callbacks = { &perf_map_jit_fini, // Cleanup function }; -#endif /* PY_HAVE_PERF_TRAMPOLINE */ \ No newline at end of file +#endif /* PY_HAVE_PERF_TRAMPOLINE */ diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 996e54b82b61e8..a2da3c7d56df50 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -230,6 +230,7 @@ perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); trampoline_api.code_padding = 0; + trampoline_api.code_alignment = 32; perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -291,7 +292,9 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; - size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); + size_t unaligned_size = code_size + trampoline_api.code_padding; + size_t chunk_size = round_up(unaligned_size, trampoline_api.code_alignment); + assert(chunk_size % trampoline_api.code_alignment == 0); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -356,7 +359,9 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, + trampoline_api.code_alignment); + assert(total_code_size % trampoline_api.code_alignment == 0); code_arena->size_left -= total_code_size; code_arena->current_addr += total_code_size; return trampoline; @@ -489,9 +494,6 @@ _PyPerfTrampoline_Init(int activate) } else { _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); - if (new_code_arena() < 0) { - return -1; - } extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { return -1; @@ -499,6 +501,9 @@ _PyPerfTrampoline_Init(int activate) if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { trampoline_api.state = trampoline_api.init_state(); } + if (new_code_arena() < 0) { + return -1; + } perf_status = PERF_STATUS_OK; } #endif diff --git a/Python/preconfig.c b/Python/preconfig.c index 5b26c75de8b3a0..e4cd10d9e3d40d 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -291,12 +291,12 @@ _PyPreConfig_InitCompatConfig(PyPreConfig *config) config->use_environment = -1; config->configure_locale = 1; - /* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) - are disabled by default using the Compat configuration. + /* gh-80624: C locale coercion (PEP 538) is disabled by default using + the Compat configuration. - Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable + Py_UTF8Mode=0 disables the UTF-8 mode. PYTHONUTF8 environment variable is ignored (even if use_environment=1). */ - config->utf8_mode = 0; + config->utf8_mode = 1; config->coerce_c_locale = 0; config->coerce_c_locale_warn = 0; @@ -317,8 +317,8 @@ PyPreConfig_InitPythonConfig(PyPreConfig *config) config->isolated = 0; config->parse_argv = 1; config->use_environment = 1; - /* Set to -1 to enable C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) - depending on the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE + /* Set to -1 to enable C locale coercion (PEP 538) depending on + the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE environment variables. */ config->coerce_c_locale = -1; config->coerce_c_locale_warn = -1; @@ -338,7 +338,7 @@ PyPreConfig_InitIsolatedConfig(PyPreConfig *config) config->configure_locale = 0; config->isolated = 1; config->use_environment = 0; - config->utf8_mode = 0; + config->utf8_mode = 1; config->dev_mode = 0; #ifdef MS_WINDOWS config->legacy_windows_fs_encoding = 0; @@ -649,23 +649,7 @@ preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) return _PyStatus_OK(); } - -#ifndef MS_WINDOWS - if (config->utf8_mode < 0) { - /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc != NULL - && (strcmp(ctype_loc, "C") == 0 - || strcmp(ctype_loc, "POSIX") == 0)) - { - config->utf8_mode = 1; - } - } -#endif - - if (config->utf8_mode < 0) { - config->utf8_mode = 0; - } + config->utf8_mode = 1; return _PyStatus_OK(); } @@ -700,7 +684,7 @@ preconfig_init_coerce_c_locale(PyPreConfig *config) /* Test if coerce_c_locale equals to -1 or equals to 1: PYTHONCOERCECLOCALE=1 doesn't imply that the C locale is always coerced. - It is only coerced if if the LC_CTYPE locale is "C". */ + It is only coerced if the LC_CTYPE locale is "C". */ if (config->coerce_c_locale < 0 || config->coerce_c_locale == 1) { /* The C locale enables the C locale coercion (PEP 538) */ if (_Py_LegacyLocaleDetected(0)) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 724fda63511282..e22a9cc1c75050 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -760,6 +760,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyDateTime_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -1702,8 +1707,10 @@ finalize_modules(PyThreadState *tstate) #endif // Stop watching __builtin__ modifications - PyDict_Unwatch(0, interp->builtins); - + if (PyDict_Unwatch(0, interp->builtins) < 0) { + // might happen if interp is cleared before watching the __builtin__ + PyErr_Clear(); + } PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -2377,15 +2384,13 @@ new_interpreter(PyThreadState **tstate_p, error: *tstate_p = NULL; if (tstate != NULL) { - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + Py_EndInterpreter(tstate); + } else { + PyInterpreterState_Delete(interp); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyInterpreterState_Delete(interp); - return status; } diff --git a/Python/pystate.c b/Python/pystate.c index 0d4c26f92cec90..04ca6edb4aaa0e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1682,6 +1682,10 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#endif + if (tstate->c_profilefunc != NULL) { tstate->interp->sys_profiling_threads--; tstate->c_profilefunc = NULL; @@ -1690,6 +1694,11 @@ PyThreadState_Clear(PyThreadState *tstate) tstate->interp->sys_tracing_threads--; tstate->c_tracefunc = NULL; } + +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#endif + Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index d1fcb478d2b035..5324a7aaa6f5e5 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -110,6 +110,14 @@ get_page_size(void) { return page_size; } +typedef struct page_cache_entry { + uintptr_t page_addr; // page-aligned base address + char *data; + int valid; + struct page_cache_entry *next; +} page_cache_entry_t; + +#define MAX_PAGES 1024 // Define a platform-independent process handle structure typedef struct { @@ -121,9 +129,27 @@ typedef struct { #elif defined(__linux__) int memfd; #endif + page_cache_entry_t pages[MAX_PAGES]; Py_ssize_t page_size; } proc_handle_t; +static void +_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + PyMem_RawFree(handle->pages[i].data); + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } +} + +UNUSED static void +_Py_RemoteDebug_ClearCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].valid = 0; + } +} #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static mach_port_t pid_to_task(pid_t pid); @@ -152,6 +178,10 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->memfd = -1; #endif handle->page_size = get_page_size(); + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } return 0; } @@ -170,6 +200,7 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { } #endif handle->pid = 0; + _Py_RemoteDebug_FreePageCache(handle); } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -720,6 +751,14 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c #ifdef MS_WINDOWS +static int is_process_alive(HANDLE hProcess) { + DWORD exitCode; + if (GetExitCodeProcess(hProcess, &exitCode)) { + return exitCode == STILL_ACTIVE; + } + return 0; +} + static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* secname) { HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { @@ -880,7 +919,9 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) _PyErr_ChainExceptions1(exc); } #else - Py_UNREACHABLE(); + _set_debug_exception_cause(PyExc_RuntimeError, + "Reading the PyRuntime section is not supported on this platform"); + return 0; #endif return address; @@ -950,6 +991,13 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address SIZE_T result = 0; do { if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) { + // Check if the process is still alive: we need to be able to tell our caller + // that the process is dead and not just that the read failed. + if (!is_process_alive(handle->hProcess)) { + _set_errno(ESRCH); + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); _set_debug_exception_cause(PyExc_OSError, @@ -982,6 +1030,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address return read_remote_memory_fallback(handle, remote_address, len, dst); } PyErr_SetFromErrno(PyExc_OSError); + if (errno == ESRCH) { + return -1; + } _set_debug_exception_cause(PyExc_OSError, "process_vm_readv failed for PID %d at address 0x%lx " "(size %zu, partial read %zd bytes): %s", @@ -1035,6 +1086,53 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, size_t size, void *out) { + size_t page_size = handle->page_size; + uintptr_t page_base = addr & ~(page_size - 1); + size_t offset_in_page = addr - page_base; + + if (offset_in_page + size > page_size) { + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); + } + + // Search for valid cached page + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (entry->valid && entry->page_addr == page_base) { + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + + // Find reusable slot + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (!entry->valid) { + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); + if (entry->data == NULL) { + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; + } + } + + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact ammount as a fallback + PyErr_Clear(); + goto fallback; + } + + entry->page_addr = page_base; + entry->valid = 1; + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + +fallback: + // Cache full โ€” fallback to uncached read return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 56e349a544c079..63e4599c31efc3 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -245,9 +245,6 @@ static const char* _Py_stdlib_module_names[] = { "socket", "socketserver", "sqlite3", -"sre_compile", -"sre_constants", -"sre_parse", "ssl", "stat", "statistics", diff --git a/Python/symtable.c b/Python/symtable.c index a3d0fff80d24a1..bcd7365f8e1f14 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2413,7 +2413,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); if (e->v.Lambda.args->kw_defaults) VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); - if (!symtable_enter_block(st, &_Py_ID(lambda), + if (!symtable_enter_block(st, &_Py_STR(anon_lambda), FunctionBlock, (void *)e, LOCATION(e))) { return 0; } @@ -3055,7 +3055,7 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, static int symtable_visit_genexp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(genexpr), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_genexpr), e->v.GeneratorExp.generators, e->v.GeneratorExp.elt, NULL); } @@ -3063,7 +3063,7 @@ symtable_visit_genexp(struct symtable *st, expr_ty e) static int symtable_visit_listcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(listcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_listcomp), e->v.ListComp.generators, e->v.ListComp.elt, NULL); } @@ -3071,7 +3071,7 @@ symtable_visit_listcomp(struct symtable *st, expr_ty e) static int symtable_visit_setcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(setcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_setcomp), e->v.SetComp.generators, e->v.SetComp.elt, NULL); } @@ -3079,7 +3079,7 @@ symtable_visit_setcomp(struct symtable *st, expr_ty e) static int symtable_visit_dictcomp(struct symtable *st, expr_ty e) { - return symtable_handle_comprehension(st, e, &_Py_ID(dictcomp), + return symtable_handle_comprehension(st, e, &_Py_STR(anon_dictcomp), e->v.DictComp.generators, e->v.DictComp.key, e->v.DictComp.value); diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c7..1e754040eaf1cc 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -66,6 +66,7 @@ def clean_lines(text): Python/dynload_dl.c # dl.h Python/dynload_hpux.c # dl.h Python/emscripten_signal.c +Python/emscripten_syscalls.c Python/thread_pthread.h Python/thread_pthread_stubs.h @@ -82,6 +83,8 @@ def clean_lines(text): Python/generated_cases.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h +# XXX: Throws errors if PY_VERSION_HEX is not mocked out +Modules/clinic/_testclinic_depr.c.h # not actually source Python/bytecodes.c diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 15b18f5286b399..dc626e4bea0f59 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -167,6 +167,8 @@ Python/sysmodule.c - _preinit_xoptions - # XXX need race protection? Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/faulthandler.c faulthandler_dump_c_stack reentrant - +Modules/grpmodule.c - group_db_mutex - +Modules/pwdmodule.c - pwd_db_mutex - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 47de205c0e9120..4c210fbf8d28e9 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -106,8 +106,9 @@ class Emitter: out: CWriter labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] + cannot_escape: bool - def __init__(self, out: CWriter, labels: dict[str, Label]): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): self._replacers = { "EXIT_IF": self.exit_if, "DEOPT_IF": self.deopt_if, @@ -127,6 +128,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): } self.out = out self.labels = labels + self.cannot_escape = cannot_escape def dispatch( self, @@ -238,7 +240,8 @@ def decref_inputs( next(tkn_iter) self._print_storage("DECREF_INPUTS", storage) try: - storage.close_inputs(self.out) + if not self.cannot_escape: + storage.close_inputs(self.out) except StackError as ex: raise analysis_error(ex.args[0], tkn) except Exception as ex: @@ -476,7 +479,7 @@ def emit_SimpleStmt( reachable = True tkn = stmt.contents[-1] try: - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not self.cannot_escape: escape = uop.properties.escaping_calls[stmt] if escape.kills is not None: self.stackref_kill(escape.kills, storage, True) @@ -513,7 +516,7 @@ def emit_SimpleStmt( self.out.emit(tkn) else: self.out.emit(tkn) - if stmt in uop.properties.escaping_calls: + if stmt in uop.properties.escaping_calls and not self.cannot_escape: self.emit_reload(storage) return reachable, None, storage except StackError as ex: diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 4556b6d5a74f37..ea9dd836d98e22 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -245,6 +245,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], original_uop: Uop, st outp.name: self.emit_stackref_override for outp in self.original_uop.stack.outputs } self._replacers = {**self._replacers, **overrides} + self.cannot_escape = True def emit_to_with_replacement( self, @@ -397,9 +398,9 @@ def generate_abstract_interpreter( out.emit("\n") base_uop_names = set([uop.name for uop in base.uops.values()]) for abstract_uop_name in abstract.uops: - assert ( - abstract_uop_name in base_uop_names - ), f"All abstract uops should override base uops, but {abstract_uop_name} is not." + if abstract_uop_name not in base_uop_names: + raise ValueError(f"All abstract uops should override base uops, " + "but {abstract_uop_name} is not.") for uop in base.uops.values(): override: Uop | None = None diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 39d0ac557a60f5..6e89e8de7cccf1 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -18,6 +18,7 @@ class BaseUnsignedIntConverter(CConverter): + bitwise = False def use_converter(self) -> None: if self.converter: @@ -25,6 +26,38 @@ def use_converter(self) -> None: f'{self.converter}()') def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if self.bitwise: + result = self.format_code(""" + {{{{ + Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) {{{{ + goto exit; + }}}} + if ((size_t)_bytes > sizeof({type})) {{{{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "integer value out of range", 1) < 0) + {{{{ + goto exit; + }}}} + }}}} + }}}} + """, + argname=argname, + type=self.type, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi)) + if self.format_unit in ('k', 'K'): + result = self.format_code(""" + if (!PyIndex_Check({argname})) {{{{ + {bad_argument} + goto exit; + }}}}""", + argname=argname, + bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi)) + result + return result + if not limited_capi: return super().parse_arg(argname, displayname, limited_capi=limited_capi) return self.format_code(""" @@ -172,13 +205,14 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st @add_legacy_c_converter('B', bitwise=True) -class unsigned_char_converter(CConverter): +class unsigned_char_converter(BaseUnsignedIntConverter): type = 'unsigned char' default_type = int format_unit = 'b' c_ignored_default = "'\0'" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'B' @@ -206,19 +240,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - elif self.format_unit == 'B': - return self.format_code(""" - {{{{ - unsigned long ival = PyLong_AsUnsignedLongMask({argname}); - if (ival == (unsigned long)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - else {{{{ - {paramname} = (unsigned char) ival; - }}}} - }}}} - """, - argname=argname) return super().parse_arg(argname, displayname, limited_capi=limited_capi) @@ -265,22 +286,12 @@ class unsigned_short_converter(BaseUnsignedIntConverter): c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'H' else: self.converter = '_PyLong_UnsignedShort_Converter' - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'H': - return self.format_code(""" - {paramname} = (unsigned short)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned short)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - @add_legacy_c_converter('C', accept={str}) class int_converter(CConverter): @@ -336,22 +347,12 @@ class unsigned_int_converter(BaseUnsignedIntConverter): c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'I' else: self.converter = '_PyLong_UnsignedInt_Converter' - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'I': - return self.format_code(""" - {paramname} = (unsigned int)PyLong_AsUnsignedLongMask({argname}); - if ({paramname} == (unsigned int)-1 && PyErr_Occurred()) {{{{ - goto exit; - }}}} - """, - argname=argname) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - class long_converter(CConverter): type = 'long' @@ -377,25 +378,12 @@ class unsigned_long_converter(BaseUnsignedIntConverter): c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'k' else: self.converter = '_PyLong_UnsignedLong_Converter' - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'k': - return self.format_code(""" - if (!PyIndex_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - class long_long_converter(CConverter): type = 'long long' @@ -421,25 +409,12 @@ class unsigned_long_long_converter(BaseUnsignedIntConverter): c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: + self.bitwise = bitwise if bitwise: self.format_unit = 'K' else: self.converter = '_PyLong_UnsignedLongLong_Converter' - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: - if self.format_unit == 'K': - return self.format_code(""" - if (!PyIndex_Check({argname})) {{{{ - {bad_argument} - goto exit; - }}}} - {paramname} = PyLong_AsUnsignedLongLongMask({argname}); - """, - argname=argname, - bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), - ) - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - class Py_ssize_t_converter(CConverter): type = 'Py_ssize_t' diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 282ff64cd33089..eca41531f7c8e9 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -877,43 +877,16 @@ def parse_parameter(self, line: str) -> None: # handle "as" for parameters too c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None + m = re.match(r'(?:\* *)?\w+( +as +(\w+))', line) + if m: + c_name = m[2] + line = line[:m.start(1)] + line[m.end(1):] + try: - ast_input = f"def x({base}): pass" + ast_input = f"def x({line}\n): pass" module = ast.parse(ast_input) except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) + fail(f"Function {self.function.name!r} has an invalid parameter declaration: {line!r}") function = module.body[0] assert isinstance(function, ast.FunctionDef) @@ -922,9 +895,6 @@ def parse_parameter(self, line: str) -> None: if len(function_args.args) > 1: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (**kwargs?): {line!r}") @@ -944,7 +914,7 @@ def parse_parameter(self, line: str) -> None: name = 'varpos_' + name value: object - if not default: + if not function_args.defaults: if is_vararg: value = NULL else: @@ -955,17 +925,13 @@ def parse_parameter(self, line: str) -> None: if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") else: - if is_vararg: - fail("Vararg can't take a default value!") + expr = function_args.defaults[0] + default = ast_input[expr.col_offset: expr.end_col_offset].strip() if self.parameter_state is ParamState.REQUIRED: self.parameter_state = ParamState.OPTIONAL - default = default.strip() bad = False - ast_input = f"x = {default}" try: - module = ast.parse(ast_input) - if 'c_default' not in kwargs: # we can only represent very simple data values in C. # detect whether default is okay, via a denylist @@ -992,13 +958,14 @@ def bad_node(self, node: ast.AST) -> None: visit_Starred = bad_node denylist = DetectBadNodes() - denylist.visit(module) + denylist.visit(expr) bad = denylist.bad else: # if they specify a c_default, we can be more lenient about the default value. # but at least make an attempt at ensuring it's a valid expression. + code = compile(ast.Expression(expr), '<expr>', 'eval') try: - value = eval(default) + value = eval(code) except NameError: pass # probably a named constant except Exception as e: @@ -1010,9 +977,6 @@ def bad_node(self, node: ast.AST) -> None: if bad: fail(f"Unsupported expression as default value: {default!r}") - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value # mild hack: explicitly support NULL as a default value c_default: str | None if isinstance(expr, ast.Name) and expr.id == 'NULL': @@ -1064,8 +1028,6 @@ def bad_node(self, node: ast.AST) -> None: else: c_default = py_default - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py index 1077e4106fdfbd..33db110b728dba 100644 --- a/Tools/jit/_optimizers.py +++ b/Tools/jit/_optimizers.py @@ -70,21 +70,21 @@ class Optimizer: path: pathlib.Path _: dataclasses.KW_ONLY - # prefix used to mangle symbols on some platforms: - prefix: str = "" + # Prefixes used to mangle local labels and symbols: + label_prefix: str + symbol_prefix: str # The first block in the linked list: _root: _Block = dataclasses.field(init=False, default_factory=_Block) _labels: dict[str, _Block] = dataclasses.field(init=False, default_factory=dict) # No groups: _re_noninstructions: typing.ClassVar[re.Pattern[str]] = re.compile( - r"\s*(?:\.|#|//|$)" + r"\s*(?:\.|#|//|;|$)" ) # One group (label): _re_label: typing.ClassVar[re.Pattern[str]] = re.compile( r'\s*(?P<label>[\w."$?@]+):' ) # Override everything that follows in subclasses: - _alignment: typing.ClassVar[int] = 1 _branches: typing.ClassVar[dict[str, str | None]] = {} # Two groups (instruction and target): _re_branch: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH @@ -131,8 +131,12 @@ def __post_init__(self) -> None: block.fallthrough = False def _preprocess(self, text: str) -> str: - # Override this method to do preprocessing of the textual assembly: - return text + # Override this method to do preprocessing of the textual assembly. + # In all cases, replace references to the _JIT_CONTINUE symbol with + # references to a local _JIT_CONTINUE label (which we will add later): + continue_symbol = rf"\b{re.escape(self.symbol_prefix)}_JIT_CONTINUE\b" + continue_label = f"{self.label_prefix}_JIT_CONTINUE" + return re.sub(continue_symbol, continue_label, text) @classmethod def _invert_branch(cls, line: str, target: str) -> str | None: @@ -197,15 +201,12 @@ def _insert_continue_label(self) -> None: # jmp FOO # After: # jmp FOO - # .balign 8 # _JIT_CONTINUE: # This lets the assembler encode _JIT_CONTINUE jumps at build time! - align = _Block() - align.noninstructions.append(f"\t.balign\t{self._alignment}") - continuation = self._lookup_label(f"{self.prefix}_JIT_CONTINUE") + continuation = self._lookup_label(f"{self.label_prefix}_JIT_CONTINUE") assert continuation.label continuation.noninstructions.append(f"{continuation.label}:") - end.link, align.link, continuation.link = align, continuation, end.link + end.link, continuation.link = continuation, end.link def _mark_hot_blocks(self) -> None: # Start with the last block, and perform a DFS to find all blocks that @@ -285,8 +286,6 @@ def run(self) -> None: class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods """aarch64-apple-darwin/aarch64-pc-windows-msvc/aarch64-unknown-linux-gnu""" - # TODO: @diegorusso - _alignment = 8 # https://developer.arm.com/documentation/ddi0602/2025-03/Base-Instructions/B--Branch- _re_jump = re.compile(r"\s*b\s+(?P<target>[\w.]+)") @@ -302,18 +301,3 @@ class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods _re_jump = re.compile(r"\s*jmp\s+(?P<target>[\w.]+)") # https://www.felixcloutier.com/x86/ret _re_return = re.compile(r"\s*ret\b") - - -class OptimizerX8664Windows(OptimizerX86): # pylint: disable = too-few-public-methods - """x86_64-pc-windows-msvc""" - - def _preprocess(self, text: str) -> str: - text = super()._preprocess(text) - # Before: - # rex64 jmpq *__imp__JIT_CONTINUE(%rip) - # After: - # jmp _JIT_CONTINUE - far_indirect_jump = ( - rf"rex64\s+jmpq\s+\*__imp_(?P<target>{self.prefix}_JIT_\w+)\(%rip\)" - ) - return re.sub(far_indirect_jump, r"jmp\t\g<target>", text) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index ed10329d25d2f9..3883671e92aa39 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -44,7 +44,8 @@ class _Target(typing.Generic[_S, _R]): _: dataclasses.KW_ONLY args: typing.Sequence[str] = () optimizer: type[_optimizers.Optimizer] = _optimizers.Optimizer - prefix: str = "" + label_prefix: typing.ClassVar[str] + symbol_prefix: typing.ClassVar[str] stable: bool = False debug: bool = False verbose: bool = False @@ -137,7 +138,15 @@ async def _compile( f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", f"-I{CPYTHON / 'Python'}", f"-I{CPYTHON / 'Tools' / 'jit'}", - "-O3", + # -O2 and -O3 include some optimizations that make sense for + # standalone functions, but not for snippets of code that are going + # to be laid out end-to-end (like ours)... common examples include + # passes like tail-duplication, or aligning jump targets with nops. + # -Os is equivalent to -O2 with many of these problematic passes + # disabled. Based on manual review, for *our* purposes it usually + # generates better code than -O2 (and -O2 usually generates better + # code than -O3). As a nice benefit, it uses less memory too: + "-Os", "-S", # Shorten full absolute file paths in the generated code (like the # __FILE__ macro and assert failure messages) for reproducibility: @@ -164,7 +173,9 @@ async def _compile( *shlex.split(self.cflags), ] await _llvm.run("clang", args_s, echo=self.verbose) - self.optimizer(s, prefix=self.prefix).run() + self.optimizer( + s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix + ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run("clang", args_o, echo=self.verbose) return await self._parse(o) @@ -266,7 +277,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = base + symbol["Value"] name = symbol["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) if name not in group.symbols: group.symbols[name] = value, offset for wrapped_relocation in section["Relocations"]: @@ -277,9 +288,9 @@ def _handle_section( def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: if name.startswith("__imp_"): name = name.removeprefix("__imp_") - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) return _stencils.HoleValue.GOT, name - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) return _stencils.symbol_to_value(name) def _handle_relocation( @@ -327,9 +338,24 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) +class _COFF32(_COFF): + # These mangle like Mach-O and other "older" formats: + label_prefix = "L" + symbol_prefix = "_" + + +class _COFF64(_COFF): + # These mangle like ELF and other "newer" formats: + label_prefix = ".L" + symbol_prefix = "" + + class _ELF( _Target[_schema.ELFSection, _schema.ELFRelocation] ): # pylint: disable = too-few-public-methods + label_prefix = ".L" + symbol_prefix = "" + def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: @@ -366,7 +392,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] name = symbol["Name"]["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) assert not section["Relocations"] @@ -401,7 +427,7 @@ def _handle_relocation( }, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s case { "Addend": addend, @@ -410,7 +436,7 @@ def _handle_relocation( "Type": {"Name": kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) case _: raise NotImplementedError(relocation) @@ -420,6 +446,9 @@ def _handle_relocation( class _MachO( _Target[_schema.MachOSection, _schema.MachORelocation] ): # pylint: disable = too-few-public-methods + label_prefix = "L" + symbol_prefix = "_" + def _handle_section( self, section: _schema.MachOSection, group: _stencils.StencilGroup ) -> None: @@ -427,10 +456,10 @@ def _handle_section( assert "SectionData" in section flags = {flag["Name"] for flag in section["Attributes"]["Flags"]} name = section["Name"]["Value"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) if "Debug" in flags: return - if "SomeInstructions" in flags: + if "PureInstructions" in flags: value = _stencils.HoleValue.CODE stencil = group.code start_address = 0 @@ -451,7 +480,7 @@ def _handle_section( symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address name = symbol["Name"]["Name"] - name = name.removeprefix(self.prefix) + name = name.removeprefix(self.symbol_prefix) group.symbols[name] = value, offset assert "Relocations" in section for wrapped_relocation in section["Relocations"]: @@ -476,7 +505,7 @@ def _handle_relocation( }, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s addend = 0 case { @@ -485,7 +514,7 @@ def _handle_relocation( "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.HoleValue.GOT, s addend = ( int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 @@ -500,7 +529,7 @@ def _handle_relocation( "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) addend = ( int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 @@ -515,7 +544,7 @@ def _handle_relocation( "Type": {"Name": kind}, }: offset += base - s = s.removeprefix(self.prefix) + s = s.removeprefix(self.symbol_prefix) value, symbol = _stencils.symbol_to_value(s) addend = 0 case _: @@ -523,19 +552,19 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -def get_target(host: str) -> _COFF | _ELF | _MachO: +def get_target(host: str) -> _COFF32 | _COFF64 | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" optimizer: type[_optimizers.Optimizer] - target: _COFF | _ELF | _MachO + target: _COFF32 | _COFF64 | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): condition = "defined(__aarch64__) && defined(__APPLE__)" optimizer = _optimizers.OptimizerAArch64 - target = _MachO(host, condition, optimizer=optimizer, prefix="_") + target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll", "-fplt"] condition = "defined(_M_ARM64)" optimizer = _optimizers.OptimizerAArch64 - target = _COFF(host, condition, args=args, optimizer=optimizer) + target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"aarch64-.*-linux-gnu", host): # -mno-outline-atomics: Keep intrinsics from being emitted. args = ["-fpic", "-mno-outline-atomics"] @@ -547,16 +576,16 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-DPy_NO_ENABLE_SHARED", "-Wno-ignored-attributes"] optimizer = _optimizers.OptimizerX86 condition = "defined(_M_IX86)" - target = _COFF(host, condition, args=args, optimizer=optimizer, prefix="_") + target = _COFF32(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-apple-darwin.*", host): condition = "defined(__x86_64__) && defined(__APPLE__)" optimizer = _optimizers.OptimizerX86 - target = _MachO(host, condition, optimizer=optimizer, prefix="_") + target = _MachO(host, condition, optimizer=optimizer) elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] condition = "defined(_M_X64)" - optimizer = _optimizers.OptimizerX8664Windows - target = _COFF(host, condition, args=args, optimizer=optimizer) + optimizer = _optimizers.OptimizerX86 + target = _COFF64(host, condition, args=args, optimizer=optimizer) elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fno-pic", "-mcmodel=medium", "-mlarge-data-threshold=0"] condition = "defined(__x86_64__) && defined(__linux__)" diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h index f767ef68127eb7..10829654eabb38 100644 --- a/Tools/jit/jit.h +++ b/Tools/jit/jit.h @@ -6,3 +6,7 @@ typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; #define PATCH_VALUE(TYPE, NAME, ALIAS) \ PyAPI_DATA(void) ALIAS; \ TYPE NAME = (TYPE)(uintptr_t)&ALIAS; + +#define DECLARE_TARGET(NAME) \ + _Py_CODEUNIT *__attribute__((preserve_none, visibility("hidden"))) \ + NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); diff --git a/Tools/jit/shim.c b/Tools/jit/shim.c index ebd4e9bc858b73..0c7feb746c9679 100644 --- a/Tools/jit/shim.c +++ b/Tools/jit/shim.c @@ -10,6 +10,6 @@ _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { // Note that this is *not* a tail call: - PATCH_VALUE(jit_func_preserve_none, call, _JIT_CONTINUE); - return call(frame, stack_pointer, tstate); + DECLARE_TARGET(_JIT_CONTINUE); + return _JIT_CONTINUE(frame, stack_pointer, tstate); } diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 5ee26f93f1e266..d07f56e9ce6b42 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -74,10 +74,10 @@ do { \ do { \ } while (0) -#define PATCH_JUMP(ALIAS) \ -do { \ - PATCH_VALUE(jit_func_preserve_none, jump, ALIAS); \ - __attribute__((musttail)) return jump(frame, stack_pointer, tstate); \ +#define PATCH_JUMP(ALIAS) \ +do { \ + DECLARE_TARGET(ALIAS); \ + __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate); \ } while (0) #undef JUMP_TO_JUMP_TARGET diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 52ae743c26b6b8..7dd56f98a652cc 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -56,7 +56,7 @@ def visit_NamedItem(self, item: NamedItem) -> None: class KeywordCollectorVisitor(GrammarVisitor): - """Visitor that collects all the keywods and soft keywords in the Grammar""" + """Visitor that collects all the keywords and soft keywords in the Grammar""" def __init__(self, gen: "ParserGenerator", keywords: Dict[str, int], soft_keywords: Set[str]): self.generator = gen diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 66898885c0a412..e5deac497fbe3f 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.111.2 +hypothesis==6.135.26 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 68cfad3f92cdc7..905af9dcfd802e 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -492,7 +492,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoned because it it too short.", + "A potential trace is abandoned because it is too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index b1a5df91901fc6..f4c8fde8346fd9 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -1,12 +1,12 @@ #!./python -"""Run Python tests against multiple installations of OpenSSL and LibreSSL +"""Run Python tests against multiple installations of cryptography libraries The script - (1) downloads OpenSSL / LibreSSL tar bundle + (1) downloads the tar bundle (2) extracts it to ./src - (3) compiles OpenSSL / LibreSSL - (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ + (3) compiles the relevant library + (4) installs that library into ../multissl/$LIB/$VERSION/ (5) forces a recompilation of Python modules using the header and library files from ../multissl/$LIB/$VERSION/ (6) runs Python's test suite @@ -61,6 +61,10 @@ LIBRESSL_RECENT_VERSIONS = [ ] +AWSLC_RECENT_VERSIONS = [ + "1.55.0", +] + # store files in ../multissl HERE = os.path.dirname(os.path.abspath(__file__)) PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) @@ -70,9 +74,9 @@ parser = argparse.ArgumentParser( prog='multissl', description=( - "Run CPython tests with multiple OpenSSL and LibreSSL " + "Run CPython tests with multiple cryptography libraries" "versions." - ) + ), ) parser.add_argument( '--debug', @@ -102,6 +106,14 @@ "OpenSSL and LibreSSL versions are given." ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) ) +parser.add_argument( + '--awslc', + nargs='+', + default=(), + help=( + "AWS-LC versions, defaults to '{}' if no crypto library versions are given." + ).format(AWSLC_RECENT_VERSIONS) +) parser.add_argument( '--tests', nargs='*', @@ -111,7 +123,7 @@ parser.add_argument( '--base-directory', default=MULTISSL_DIR, - help="Base directory for OpenSSL / LibreSSL sources and builds." + help="Base directory for crypto library sources and builds." ) parser.add_argument( '--no-network', @@ -124,8 +136,8 @@ choices=['library', 'modules', 'tests'], default='tests', help=( - "Which steps to perform. 'library' downloads and compiles OpenSSL " - "or LibreSSL. 'module' also compiles Python modules. 'tests' builds " + "Which steps to perform. 'library' downloads and compiles a crypto" + "library. 'module' also compiles Python modules. 'tests' builds " "all and runs the test suite." ) ) @@ -453,6 +465,34 @@ class BuildLibreSSL(AbstractBuilder): build_template = "libressl-{}" +class BuildAWSLC(AbstractBuilder): + library = "AWS-LC" + url_templates = ( + "https://github.com/aws/aws-lc/archive/refs/tags/v{v}.tar.gz", + ) + src_template = "aws-lc-{}.tar.gz" + build_template = "aws-lc-{}" + + def _build_src(self, config_args=()): + cwd = self.build_dir + log.info("Running build in {}".format(cwd)) + env = os.environ.copy() + env["LD_RUN_PATH"] = self.lib_dir # set rpath + if self.system: + env['SYSTEM'] = self.system + cmd = [ + "cmake", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_PREFIX_PATH={}".format(self.install_dir), + "-DCMAKE_INSTALL_PREFIX={}".format(self.install_dir), + "-DBUILD_SHARED_LIBS=ON", + "-DBUILD_TESTING=OFF", + "-DFIPS=OFF", + ] + self._subprocess_call(cmd, cwd=cwd, env=env) + self._subprocess_call(["make", "-j{}".format(self.jobs)], cwd=cwd, env=env) + + def configure_make(): if not os.path.isfile('Makefile'): log.info('Running ./configure') @@ -467,9 +507,10 @@ def configure_make(): def main(): args = parser.parse_args() - if not args.openssl and not args.libressl: + if not args.openssl and not args.libressl and not args.awslc: args.openssl = list(OPENSSL_RECENT_VERSIONS) args.libressl = list(LIBRESSL_RECENT_VERSIONS) + args.awslc = list(AWSLC_RECENT_VERSIONS) if not args.disable_ancient: args.openssl.extend(OPENSSL_OLD_VERSIONS) args.libressl.extend(LIBRESSL_OLD_VERSIONS) @@ -496,22 +537,15 @@ def main(): # download and register builder builds = [] - - for version in args.openssl: - build = BuildOpenSSL( - version, - args - ) - build.install() - builds.append(build) - - for version in args.libressl: - build = BuildLibreSSL( - version, - args - ) - build.install() - builds.append(build) + for build_class, versions in [ + (BuildOpenSSL, args.openssl), + (BuildLibreSSL, args.libressl), + (BuildAWSLC, args.awslc), + ]: + for version in versions: + build = build_class(version, args) + build.install() + builds.append(build) if args.steps in {'modules', 'tests'}: for build in builds: @@ -539,7 +573,7 @@ def main(): else: print('Executed all SSL tests.') - print('OpenSSL / LibreSSL versions:') + print('OpenSSL / LibreSSL / AWS-LC versions:') for build in builds: print(" * {0.library} {0.version}".format(build)) diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 232321c515721e..efe9a3550c3e19 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -22,7 +22,7 @@ https://github.com/psf/webassembly for more information. ### Build To cross compile to the ``wasm32-emscripten`` platform you need -[the Emscripten compiler toolchain](https://emscripten.org/), +[the Emscripten compiler toolchain](https://emscripten.org/), a Python interpreter, and an installation of Node version 18 or newer. Emscripten version 4.0.2 is recommended; newer versions may also work, but all official testing is performed with that version. All commands below are relative @@ -84,13 +84,24 @@ make a node application that "embeds" the interpreter instead of acting like the CLI you will need to write your own alternative to `node_entry.mjs`. +### Running tests + +After building, you can run the full test suite with: +```shell +./cross-build/wasm32-emscripten/build/python/python.sh -m test -uall +``` +You can run the browser smoke test with: +```shell +./Tools/wasm/emscripten/browser_test/run_test.sh +``` + ### The Web Example -When building for Emscripten, the web example will be built automatically. It is -in the ``web_example`` directory. To run the web example, ``cd`` into the +When building for Emscripten, the web example will be built automatically. It +is in the ``web_example`` directory. To run the web example, ``cd`` into the ``web_example`` directory, then run ``python server.py``. This will start a web -server; you can then visit ``http://localhost:8000/python.html`` in a browser to -see a simple REPL example. +server; you can then visit ``http://localhost:8000/`` in a browser to see a +simple REPL example. The web example relies on a bug fix in Emscripten version 3.1.73 so if you build with earlier versions of Emscripten it may not work. The web example uses diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index c0d58aeaadd2cf..202dd298199b56 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -8,6 +8,7 @@ import subprocess import sys import sysconfig +import hashlib import tempfile from urllib.request import urlopen from pathlib import Path @@ -164,20 +165,60 @@ def make_build_python(context, working_dir): print(f"๐ŸŽ‰ {binary} {version}") -@subdir(HOST_BUILD_DIR, clean_ok=True) -def make_emscripten_libffi(context, working_dir): - shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) +def check_shasum(file: str, expected_shasum: str): + with open(file, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + if digest.hexdigest() != expected_shasum: + raise RuntimeError(f"Unexpected shasum for {file}") + + +def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete_on_close=False) as tmp_file: - with urlopen( - "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" - ) as response: + with urlopen(url) as response: shutil.copyfileobj(response, tmp_file) tmp_file.close() + check_shasum(tmp_file.name, expected_shasum) shutil.unpack_archive(tmp_file.name, working_dir) + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + ver = "3.4.6" + libffi_dir = working_dir / f"libffi-{ver}" + shutil.rmtree(libffi_dir, ignore_errors=True) + download_and_unpack(working_dir, f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e") call( [EMSCRIPTEN_DIR / "make_libffi.sh"], env=updated_env({"PREFIX": PREFIX_DIR}), - cwd=working_dir / "libffi-3.4.6", + cwd=libffi_dir, + quiet=context.quiet, + ) + + +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_mpdec(context, working_dir): + ver = "4.0.1" + mpdec_dir = working_dir / f"mpdecimal-{ver}" + shutil.rmtree(mpdec_dir, ignore_errors=True) + download_and_unpack(working_dir, f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8") + call( + [ + "emconfigure", + mpdec_dir / "configure", + "CFLAGS=-fPIC", + "--prefix", + PREFIX_DIR, + "--disable-shared", + ], + cwd=mpdec_dir, + quiet=context.quiet, + ) + call( + [ + "make", + "install" + ], + cwd=mpdec_dir, quiet=context.quiet, ) @@ -186,7 +227,7 @@ def make_emscripten_libffi(context, working_dir): def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" config_site = os.fsdecode( - CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" + EMSCRIPTEN_DIR / "config.site-wasm32-emscripten" ) emscripten_build_dir = working_dir.relative_to(CHECKOUT) @@ -206,6 +247,17 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner + if node_version := os.environ.get("PYTHON_NODE_VERSION", None): + res = subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm which {node_version}", + ], + text=True, + capture_output=True, + ) + host_runner = res.stdout.strip() pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() env_additions = { "CONFIG_SITE": config_site, @@ -263,10 +315,20 @@ def configure_emscripten_python(context, working_dir): REALPATH=abs_path fi + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) @@ -294,6 +356,7 @@ def build_all(context): configure_build_python, make_build_python, make_emscripten_libffi, + make_mpdec, configure_emscripten_python, make_emscripten_python, ] @@ -322,6 +385,9 @@ def main(): configure_build = subcommands.add_parser( "configure-build-python", help="Run `configure` for the " "build Python" ) + make_mpdec_cmd = subcommands.add_parser( + "make-mpdec", help="Clone mpdec repo, configure and build it for emscripten" + ) make_libffi_cmd = subcommands.add_parser( "make-libffi", help="Clone libffi repo, configure and build it for emscripten" ) @@ -342,6 +408,7 @@ def main(): build, configure_build, make_libffi_cmd, + make_mpdec_cmd, make_build, configure_host, make_host, @@ -379,6 +446,7 @@ def main(): dispatch = { "make-libffi": make_emscripten_libffi, + "make-mpdec": make_mpdec, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, diff --git a/Tools/wasm/emscripten/browser_test/.gitignore b/Tools/wasm/emscripten/browser_test/.gitignore new file mode 100644 index 00000000000000..3d54eab82a0d71 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/.gitignore @@ -0,0 +1,4 @@ +node_modules +test-results +playwright-report +test_log.txt diff --git a/Tools/wasm/emscripten/browser_test/index.spec.ts b/Tools/wasm/emscripten/browser_test/index.spec.ts new file mode 100644 index 00000000000000..2594b73022bcc1 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/index.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('/'); + + await expect(page).toHaveTitle("Emscripten PyRepl Example"); + const xterm = await page.locator('css=#terminal'); + await expect(xterm).toHaveText(/Python.*on emscripten.*Type.*for more information/); + const xtermInput = await page.getByRole('textbox'); + await xtermInput.pressSequentially(`def f():\nprint("hello", "emscripten repl!")\n\n`); + await xtermInput.pressSequentially(`f()\n`); + await expect(xterm).toHaveText(/hello emscripten repl!/); + +}); + diff --git a/Tools/wasm/emscripten/browser_test/package-lock.json b/Tools/wasm/emscripten/browser_test/package-lock.json new file mode 100644 index 00000000000000..044e3c19ce15f7 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/package-lock.json @@ -0,0 +1,672 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "browser_test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@playwright/test": "^1.54.1", + "@types/node": "^24.1.0", + "http-server": "^14.1.1", + "playwright": "^1.54.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.54.1.tgz", + "integrity": "sha512-FS8hQ12acieG2dYSksmLOF7BNxnVf2afRJdCuM1eMSxj6QTSE6G4InGF7oApGgDb65MX7AwMVlIkpru0yZA4Xw==", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.54.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "24.1.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz", + "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.8.0" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-server": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", + "license": "MIT", + "dependencies": { + "basic-auth": "^2.0.1", + "chalk": "^4.1.2", + "corser": "^2.0.1", + "he": "^1.2.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy": "^1.18.1", + "mime": "^1.6.0", + "minimist": "^1.2.6", + "opener": "^1.5.1", + "portfinder": "^1.0.28", + "secure-compare": "3.0.1", + "union": "~0.5.0", + "url-join": "^4.0.1" + }, + "bin": { + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "license": "(WTFPL OR MIT)", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/playwright": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.54.1.tgz", + "integrity": "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.54.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.54.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.54.1.tgz", + "integrity": "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/portfinder": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.37.tgz", + "integrity": "sha512-yuGIEjDAYnnOex9ddMnKZEMFE0CcGo6zbfzDklkmT1m5z734ss6JMzN9rNB3+RR7iS+F10D4/BVIaXOyh8PQKw==", + "license": "MIT", + "dependencies": { + "async": "^3.2.6", + "debug": "^4.3.6" + }, + "engines": { + "node": ">= 10.12" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", + "license": "MIT" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/undici-types": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", + "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "license": "MIT" + }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, + "node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" + } + } + } +} diff --git a/Tools/wasm/emscripten/browser_test/package.json b/Tools/wasm/emscripten/browser_test/package.json new file mode 100644 index 00000000000000..3320d4cccd0594 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/package.json @@ -0,0 +1,18 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "@playwright/test": "^1.54.1", + "@types/node": "^24.1.0", + "http-server": "^14.1.1", + "playwright": "^1.54.1" + } +} diff --git a/Tools/wasm/emscripten/browser_test/playwright.config.ts b/Tools/wasm/emscripten/browser_test/playwright.config.ts new file mode 100644 index 00000000000000..81d53ce11cb050 --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/playwright.config.ts @@ -0,0 +1,22 @@ +import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: '.', + forbidOnly: true, + retries: 2, + reporter: process.env.CI ? 'dot' : 'html', + use: { + baseURL: 'http://localhost:8787', + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: { + command: 'npx http-server ../../../../cross-build/wasm32-emscripten/build/python/web_example_pyrepl_jspi/ -p 8787', + url: 'http://localhost:8787', + }, +}); diff --git a/Tools/wasm/emscripten/browser_test/run_test.sh b/Tools/wasm/emscripten/browser_test/run_test.sh new file mode 100755 index 00000000000000..9166e0d740585e --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/run_test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" +rm -f test_log.txt +echo "Installing node packages" | tee test_log.txt +npm ci >> test_log.txt 2>&1 +echo "Installing playwright browsers" | tee test_log.txt +npx playwright install 2>> test_log.txt +echo "Running tests" | tee test_log.txt +CI=1 npx playwright test | tee test_log.txt diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/emscripten/config.site-wasm32-emscripten similarity index 97% rename from Tools/wasm/config.site-wasm32-emscripten rename to Tools/wasm/emscripten/config.site-wasm32-emscripten index 1471546a5eec17..9f98e3f3c3bb1f 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/emscripten/config.site-wasm32-emscripten @@ -1,6 +1,6 @@ # config.site override for cross compiling to wasm32-emscripten platform # -# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ +# CONFIG_SITE=Tools/wasm/emscripten/config.site-wasm32-emscripten \ # emconfigure ./configure --host=wasm32-unknown-emscripten --build=... # # Written by Christian Heimes <christian@python.org> @@ -69,7 +69,6 @@ ac_cv_func_posix_fallocate=no # Syscalls that resulted in a segfault ac_cv_func_utimensat=no -ac_cv_header_sys_ioctl_h=no # sockets are supported, but only AF_INET / AF_INET6 in non-blocking mode. # Disable AF_UNIX and AF_PACKET support, see socketmodule.h. diff --git a/Tools/wasm/emscripten/web_example/wasm_assets.py b/Tools/wasm/emscripten/wasm_assets.py similarity index 98% rename from Tools/wasm/emscripten/web_example/wasm_assets.py rename to Tools/wasm/emscripten/wasm_assets.py index deeb9229a4412b..b08e7ce1114a4a 100755 --- a/Tools/wasm/emscripten/web_example/wasm_assets.py +++ b/Tools/wasm/emscripten/wasm_assets.py @@ -18,7 +18,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parents[4].absolute() +SRCDIR = pathlib.Path(__file__).parents[3].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -84,7 +84,6 @@ "_json": ["json/"], "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], "pyexpat": ["xml/", "xmlrpc/"], - "readline": ["rlcompleter.py"], "_sqlite3": ["sqlite3/"], "_ssl": ["ssl.py"], "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/index.html similarity index 52% rename from Tools/wasm/emscripten/web_example/python.html rename to Tools/wasm/emscripten/web_example/index.html index 078f86eb764419..9c89c9c0ed3bf5 100644 --- a/Tools/wasm/emscripten/web_example/python.html +++ b/Tools/wasm/emscripten/web_example/index.html @@ -1,31 +1,39 @@ <!doctype html> <html lang="en"> <head> - <meta charset="UTF-8" /> - <meta http-equiv="X-UA-Compatible" content="IE=edge" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="author" content="Katie Bell" /> - <meta name="description" content="Simple REPL for Python WASM" /> + <meta charset="UTF-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="author" content="Katie Bell, Adam Hartz"> + <meta name="description" content="Simple REPL for Python WASM"> <title>wasm-python terminal + > + + +

+ + + diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs new file mode 100644 index 00000000000000..5642372c9d2472 --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs @@ -0,0 +1,194 @@ +// Much of this is adapted from here: +// https://github.com/mame/xterm-pty/blob/main/emscripten-pty.js +// Thanks to xterm-pty for making this possible! + +import createEmscriptenModule from "./python.mjs"; +import { openpty } from "https://unpkg.com/xterm-pty/index.mjs"; +import "https://unpkg.com/@xterm/xterm/lib/xterm.js"; + +var term = new Terminal(); +term.open(document.getElementById("terminal")); +const { master, slave: PTY } = openpty(); +term.loadAddon(master); +globalThis.PTY = PTY; + +async function setupStdlib(Module) { + const versionInt = Module.HEAPU32[Module._Py_Version >>> 2]; + const major = (versionInt >>> 24) & 0xff; + const minor = (versionInt >>> 16) & 0xff; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); +} + +const tty_ops = { + ioctl_tcgets: () => { + const termios = PTY.ioctl("TCGETS"); + const data = { + c_iflag: termios.iflag, + c_oflag: termios.oflag, + c_cflag: termios.cflag, + c_lflag: termios.lflag, + c_cc: termios.cc, + }; + return data; + }, + + ioctl_tcsets: (_tty, _optional_actions, data) => { + PTY.ioctl("TCSETS", { + iflag: data.c_iflag, + oflag: data.c_oflag, + cflag: data.c_cflag, + lflag: data.c_lflag, + cc: data.c_cc, + }); + return 0; + }, + + ioctl_tiocgwinsz: () => PTY.ioctl("TIOCGWINSZ").reverse(), + + get_char: () => { + throw new Error("Should not happen"); + }, + put_char: () => { + throw new Error("Should not happen"); + }, + + fsync: () => {}, +}; + +const POLLIN = 1; +const POLLOUT = 4; + +const waitResult = { + READY: 0, + SIGNAL: 1, + TIMEOUT: 2, +}; + +function onReadable() { + var handle; + var promise = new Promise((resolve) => { + handle = PTY.onReadable(() => resolve(waitResult.READY)); + }); + return [promise, handle]; +} + +function onSignal() { + // TODO: signal handling + var handle = { dispose() {} }; + var promise = new Promise((resolve) => {}); + return [promise, handle]; +} + +function onTimeout(timeout) { + var id; + var promise = new Promise((resolve) => { + if (timeout > 0) { + id = setTimeout(resolve, timeout, waitResult.TIMEOUT); + } + }); + var handle = { + dispose() { + if (id) { + clearTimeout(id); + } + }, + }; + return [promise, handle]; +} + +async function waitForReadable(timeout) { + let p1, p2, p3; + let h1, h2, h3; + try { + [p1, h1] = onReadable(); + [p2, h2] = onTimeout(timeout); + [p3, h3] = onSignal(); + return await Promise.race([p1, p2, p3]); + } finally { + h1.dispose(); + h2.dispose(); + h3.dispose(); + } +} + +const FIONREAD = 0x541b; + +const tty_stream_ops = { + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + let readBytes = PTY.read(length); + if (length && !readBytes.length) { + const status = await waitForReadable(-1); + if (status === waitResult.READY) { + readBytes = PTY.read(length); + } else { + throw new Error("Not implemented"); + } + } + buffer.set(readBytes, offset); + return readBytes.length; + }, + + write: (stream, buffer, offset, length) => { + // Note: default `buffer` is for some reason `HEAP8` (signed), while we want unsigned `HEAPU8`. + buffer = new Uint8Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength, + ); + const toWrite = Array.from(buffer.subarray(offset, offset + length)); + PTY.write(toWrite); + return length; + }, + + async pollAsync(stream, timeout) { + if (!PTY.readable && timeout) { + await waitForReadable(timeout); + } + return (PTY.readable ? POLLIN : 0) | (PTY.writable ? POLLOUT : 0); + }, + ioctl(stream, request, varargs) { + if (request === FIONREAD) { + const res = PTY.fromLdiscToUpperBuffer.length; + Module.HEAPU32[varargs / 4] = res; + return 0; + } + throw new Error("Unimplemented ioctl request"); + }, +}; + +async function setupStdio(Module) { + Object.assign(Module.TTY.default_tty_ops, tty_ops); + Object.assign(Module.TTY.stream_ops, tty_stream_ops); +} + +const emscriptenSettings = { + async preRun(Module) { + Module.addRunDependency("pre-run"); + Module.ENV.TERM = "xterm-256color"; + // Uncomment next line to turn on tracing (messages go to browser console). + // Module.ENV.PYREPL_TRACE = "1"; + + // Leak module so we can try to show traceback if we crash on startup + globalThis.Module = Module; + await Promise.all([setupStdlib(Module), setupStdio(Module)]); + Module.removeRunDependency("pre-run"); + }, +}; + +try { + await createEmscriptenModule(emscriptenSettings); +} catch (e) { + // Show JavaScript exception and traceback + console.warn(e); + // Show Python exception and traceback + Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + process.exit(1); +} diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 54ccc95157d57d..f3c97ff3fd11a0 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -23,7 +23,8 @@ BUILD_DIR = CROSS_BUILD_DIR / "build" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") +LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" + "# Required to statically build extension modules.").encode("utf-8") WASMTIME_VAR_NAME = "WASMTIME" WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" @@ -141,9 +142,12 @@ def build_python_is_pydebug(): def configure_build_python(context, working_dir): """Configure the build/host Python.""" if LOCAL_SETUP.exists(): - print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...") + else: + print(f"โš ๏ธ {LOCAL_SETUP} exists, but has unexpected contents") else: - print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...") + print(f"๐Ÿ“ Creating {LOCAL_SETUP} ...") LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] @@ -297,9 +301,8 @@ def clean_contents(context): shutil.rmtree(CROSS_BUILD_DIR) if LOCAL_SETUP.exists(): - with LOCAL_SETUP.open("rb") as file: - if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: - print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...") def main(): diff --git a/configure b/configure index 75ae1699a8e451..74df430d10d6bd 100755 --- a/configure +++ b/configure @@ -7254,6 +7254,8 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-linux-android/clang) : PY_SUPPORT_TIER=3 ;; #( + wasm32-*-emscripten/emcc) : + PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 @@ -9601,9 +9603,10 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY" as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" + as_fn_append LINKFORSHARED " -sTEXTDECODER=2" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -13813,6 +13816,8 @@ case $PLATFORM_TRIPLET in #( perf_trampoline=yes ;; #( aarch64-linux-gnu) : perf_trampoline=yes ;; #( + darwin) : + perf_trampoline=yes ;; #( *) : perf_trampoline=no ;; @@ -15750,10 +15755,18 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-system-libmpdec" >&5 printf %s "checking for --with-system-libmpdec... " >&6; } + # Check whether --with-system_libmpdec was given. if test ${with_system_libmpdec+y} then : - withval=$with_system_libmpdec; + withval=$with_system_libmpdec; if test "x$with_system_libmpdec" = xno +then : + LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no +fi else case e in #( e) with_system_libmpdec="yes" ;; esac @@ -15762,8 +15775,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 printf "%s\n" "$with_system_libmpdec" >&6; } - - if test "x$with_system_libmpdec" = xyes then : @@ -15841,13 +15852,6 @@ else printf "%s\n" "yes" >&6; } fi -else case e in #( - e) LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no ;; -esac fi if test "x$with_system_libmpdec" = xyes @@ -15894,21 +15898,6 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS -else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 -printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} ;; -esac -fi - -if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 -printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no fi # Disable forced inlining in debug builds, see GH-94847 @@ -19074,7 +19063,7 @@ PLATFORM_OBJS= case $ac_sys_system in #( Emscripten) : - as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o' + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_syscalls.o' as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h' ;; #( *) : @@ -28234,7 +28223,7 @@ printf "%s\n" "$ac_cv_normalize_century" >&6; } if test "$ac_cv_normalize_century" = yes then -printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h +printf "%s\n" "#define _Py_NORMALIZE_CENTURY 1" >>confdefs.h fi @@ -30862,8 +30851,8 @@ main (void) OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ; @@ -31193,9 +31182,7 @@ case $ac_sys_system in #( - py_cv_module_fcntl=n/a py_cv_module_readline=n/a - py_cv_module_termios=n/a py_cv_module_=n/a ;; #( @@ -33194,6 +33181,18 @@ fi printf "%s\n" "$py_cv_module__decimal" >&6; } +if test "x$with_system_libmpdec" = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&2;} +fi +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _dbm" >&5 printf %s "checking for stdlib extension module _dbm... " >&6; } if test "$py_cv_module__dbm" != "n/a" diff --git a/configure.ac b/configure.ac index 4da1ba78b54b0d..29e36f217402f1 100644 --- a/configure.ac +++ b/configure.ac @@ -1208,27 +1208,28 @@ dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. dnl AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], - [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc - [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version - [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC - [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - - [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang - [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], - [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang - [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang - - [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC - [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc - [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang - [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 - [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 - [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 - [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 - [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc + [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version + [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC + [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC + + [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang + [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], + [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang + [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang + + [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC + [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc + [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang + [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc + [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [wasm32-*-emscripten/emcc], [PY_SUPPORT_TIER=3], dnl Emscripten [PY_SUPPORT_TIER=0] ) @@ -2334,9 +2335,11 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) + dnl Avoid bugs in JS fallback string decoding path + AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -3689,12 +3692,13 @@ case "$ac_sys_system" in esac AC_MSG_RESULT([$SHLIBS]) -dnl perf trampoline is Linux specific and requires an arch-specific +dnl perf trampoline is Linux and macOS specific and requires an arch-specific dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], [aarch64-linux-gnu], [perf_trampoline=yes], + [darwin], [perf_trampoline=yes], [perf_trampoline=no] ) AC_MSG_RESULT([$perf_trampoline]) @@ -4169,31 +4173,30 @@ fi # Check for use of the system libmpdec library AC_MSG_CHECKING([for --with-system-libmpdec]) +AC_DEFUN([USE_BUNDLED_LIBMPDEC], + [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" + LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" + LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" + have_mpdec=yes + with_system_libmpdec=no]) AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], - [], + [AS_IF([test "x$with_system_libmpdec" = xno], + [USE_BUNDLED_LIBMPDEC()])], [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) -AC_DEFUN([USE_BUNDLED_LIBMPDEC], - [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no]) - AS_VAR_IF( [with_system_libmpdec], [yes], [PKG_CHECK_MODULES( [LIBMPDEC], [libmpdec >= 2.5.0], [], [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} - LIBMPDEC_INTERNAL=])], - [USE_BUNDLED_LIBMPDEC()]) + LIBMPDEC_INTERNAL=])]) AS_VAR_IF([with_system_libmpdec], [yes], [WITH_SAVE_ENV([ @@ -4209,16 +4212,7 @@ AS_VAR_IF([with_system_libmpdec], [yes], ], [const char *x = mpd_version();])], [have_mpdec=yes], [have_mpdec=no]) - ])], - [AC_MSG_WARN([m4_normalize([ - the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; - consider using a system installed mpdecimal library.])])]) - -AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], - [AC_MSG_WARN([m4_normalize([ - no system libmpdecimal found; falling back to bundled libmpdecimal - (deprecated and scheduled for removal in Python 3.15)])]) - USE_BUNDLED_LIBMPDEC()]) + ])]) # Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF( @@ -5128,7 +5122,7 @@ PLATFORM_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ - AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o']) + AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_syscalls.o']) AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h']) ], ) @@ -6813,7 +6807,7 @@ int main(void) [ac_cv_normalize_century=yes])]) if test "$ac_cv_normalize_century" = yes then - AC_DEFINE([Py_NORMALIZE_CENTURY], [1], + AC_DEFINE([_Py_NORMALIZE_CENTURY], [1], [Define if year with century should be normalized for strftime.]) fi @@ -7538,8 +7532,8 @@ WITH_SAVE_ENV([ ], [ OBJ_nid2sn(NID_md5); OBJ_nid2sn(NID_sha1); + OBJ_nid2sn(NID_sha512); OBJ_nid2sn(NID_sha3_512); - OBJ_nid2sn(NID_blake2b512); EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0); ])], [ac_cv_working_openssl_hashlib=yes], [ac_cv_working_openssl_hashlib=no]) ]) @@ -7775,9 +7769,7 @@ AS_CASE([$ac_sys_system], ) dnl fcntl, readline, and termios are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( - [fcntl], [readline], - [termios], ) ], [WASI], [ @@ -8136,6 +8128,16 @@ PY_STDLIB_MOD([_curses_panel], PY_STDLIB_MOD([_decimal], [], [test "$have_mpdec" = "yes"], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) + +AS_VAR_IF([with_system_libmpdec], [no], + [AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdec is scheduled for removal in Python 3.16; + consider using a system installed mpdecimal library.])])]) +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([m4_normalize([ + no system libmpdec found; falling back to pure-Python version + for the decimal module])])]) + PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) diff --git a/iOS/README.rst b/iOS/README.rst index 13b885144932e4..f0979ba152eb20 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -196,7 +196,7 @@ simulator build with a deployment target of 15.4. Merge thin frameworks into fat frameworks ----------------------------------------- -Once you've built a ``Python.framework`` for each ABI and and architecture, you +Once you've built a ``Python.framework`` for each ABI and architecture, you must produce a "fat" framework for each ABI that contains all the architectures for that ABI. diff --git a/pyconfig.h.in b/pyconfig.h.in index d7c496fccc682c..1c533b2bfb7fb4 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1737,9 +1737,6 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM -/* Define if year with century should be normalized for strftime. */ -#undef Py_NORMALIZE_CENTURY - /* Define if you want to enable remote debugging support. */ #undef Py_REMOTE_DEBUG @@ -2026,6 +2023,9 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if year with century should be normalized for strftime. */ +#undef _Py_NORMALIZE_CENTURY + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT