diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index fcfac85ed9c442..56caf0bc5256fa 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +trigger: ['main', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 steps: - template: ./docs-steps.yml @@ -35,14 +31,16 @@ jobs: - job: macOS_CI_Tests displayName: macOS CI Tests dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + # bpo-39837: macOS tests on Azure Pipelines are disabled + condition: false variables: testRunTitle: '$(build.sourceBranchName)-macos' testRunPlatform: macos pool: - vmImage: xcode9-macos10.13 + vmImage: macos-10.15 steps: - template: ./macos-steps.yml @@ -54,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1c + openssl_version: 1.1.1u steps: - template: ./posix-steps.yml @@ -67,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_CI_Tests - displayName: ManyLinux1 CI Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(build.sourceBranchName)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_CI_Tests displayName: Ubuntu CI Tests (coverage) dependsOn: Prebuild @@ -111,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1c + openssl_version: 1.1.1u steps: - template: ./posix-steps.yml @@ -131,13 +98,13 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 strategy: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(Build.SourceBranchName)-win32' testRunPlatform: win32 win64: @@ -145,7 +112,7 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(Build.SourceBranchName)-win64' testRunPlatform: win64 - maxParallel: 2 + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index 492e4e34bb2dab..ae75339b57a1d6 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,11 +12,12 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme +- script: python -m pip install -r requirements.txt + workingDirectory: '$(build.sourcesDirectory)/Doc' displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: - - script: make check suspicious html PYTHON=python + - script: make check html PYTHON=python workingDirectory: '$(build.sourcesDirectory)/Doc' displayName: 'Build documentation' @@ -31,7 +32,7 @@ steps: - ${{ if eq(parameters.upload, 'true') }}: - task: PublishBuildArtifacts@1 displayName: 'Publish docs' - + inputs: PathToPublish: '$(build.sourcesDirectory)/Doc/build' ArtifactName: docs diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml index 647081689454ac..fa38a0df8c87b8 100644 --- a/.azure-pipelines/macos-steps.yml +++ b/.azure-pipelines/macos-steps.yml @@ -6,7 +6,7 @@ steps: - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev displayName: 'Configure CPython (debug)' -- script: make -s -j4 +- script: make -j4 displayName: 'Build CPython' - script: make pythoninfo @@ -14,6 +14,8 @@ steps: - script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" displayName: 'Tests' + continueOnError: true + timeoutInMinutes: 30 - task: PublishTestResults@2 displayName: 'Publish Test Results' diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml index 3ed3abd02a7146..a63659fa204910 100644 --- a/.azure-pipelines/posix-steps.yml +++ b/.azure-pipelines/posix-steps.yml @@ -20,7 +20,7 @@ steps: - script: ./configure --with-pydebug displayName: 'Configure CPython (debug)' -- script: make -s -j4 +- script: make -j4 displayName: 'Build CPython' - ${{ if eq(parameters.coverage, 'true') }}: diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 2486f88a63fb15..a882129ac4ecee 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +pr: ['main', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 steps: - template: ./docs-steps.yml @@ -33,14 +29,16 @@ jobs: - job: macOS_PR_Tests displayName: macOS PR Tests dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) + # bpo-39837: macOS tests on Azure Pipelines are disabled + condition: false variables: testRunTitle: '$(system.pullRequest.TargetBranch)-macos' testRunPlatform: macos pool: - vmImage: xcode9-macos10.13 + vmImage: macos-10.15 steps: - template: ./macos-steps.yml @@ -54,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1c + openssl_version: 1.1.1u steps: - template: ./posix-steps.yml @@ -67,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_PR_Tests - displayName: ManyLinux1 PR Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_PR_Tests displayName: Ubuntu PR Tests (coverage) dependsOn: Prebuild @@ -111,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-22.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1c + openssl_version: 1.1.1u steps: - template: ./posix-steps.yml @@ -131,13 +98,13 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 strategy: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(System.PullRequest.TargetBranch)-win32' testRunPlatform: win32 win64: @@ -145,7 +112,10 @@ jobs: buildOpt: '-p x64' testRunTitle: '$(System.PullRequest.TargetBranch)-win64' testRunPlatform: win64 - maxParallel: 2 + winarm64: + arch: arm64 + buildOpt: '-p arm64' + maxParallel: 4 steps: - template: ./windows-steps.yml diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml new file mode 100644 index 00000000000000..3d072e3b43e17e --- /dev/null +++ b/.azure-pipelines/windows-release.yml @@ -0,0 +1,129 @@ +name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + __RealSigningCertificate: 'Python Software Foundation' +# QUEUE TIME VARIABLES +# GitRemote: python +# SourceTag: +# DoPGO: true +# SigningCertificate: 'Python Software Foundation' +# SigningDescription: 'Built: $(Build.BuildNumber)' +# DoLayout: true +# DoMSIX: true +# DoNuget: true +# DoEmbed: true +# DoMSI: true +# DoPublish: false +# PyDotOrgUsername: '' +# PyDotOrgServer: '' +# BuildToPublish: '' + +trigger: none +pr: none + +stages: +- stage: Build + displayName: Build binaries + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-build.yml + +- stage: Sign + displayName: Sign binaries + dependsOn: Build + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-sign.yml + +- stage: Layout + displayName: Generate layouts + dependsOn: Sign + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-layout-full.yml + - template: windows-release/stage-layout-embed.yml + - template: windows-release/stage-layout-nuget.yml + +- stage: Pack + dependsOn: Layout + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-nuget.yml + +- stage: Test + dependsOn: Pack + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-embed.yml + - template: windows-release/stage-test-nuget.yml + +- stage: Layout_MSIX + displayName: Generate MSIX layouts + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-layout-msix.yml + +- stage: Pack_MSIX + displayName: Package MSIX + dependsOn: Layout_MSIX + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-pack-msix.yml + +- stage: Build_MSI + displayName: Build MSI installer + dependsOn: Sign + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-msi.yml + +- stage: Test_MSI + displayName: Test MSI installer + dependsOn: Build_MSI + condition: and(succeeded(), not(variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-test-msi.yml + +- stage: PublishPyDotOrg + displayName: Publish to python.org + dependsOn: ['Test_MSI', 'Test'] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishNuget + displayName: Publish to nuget.org + dependsOn: Test + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishStore + displayName: Publish to Store + dependsOn: Pack_MSIX + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish']))) + jobs: + - template: windows-release/stage-publish-store.yml + + +- stage: PublishExistingPyDotOrg + displayName: Publish existing build to python.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-pythonorg.yml + +- stage: PublishExistingNuget + displayName: Publish existing build to nuget.org + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-nugetorg.yml + +- stage: PublishExistingStore + displayName: Publish existing build to Store + dependsOn: [] + condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish'])) + jobs: + - template: windows-release/stage-publish-store.yml diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml new file mode 100644 index 00000000000000..5ca2016d65f9e1 --- /dev/null +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -0,0 +1,84 @@ +parameters: + ShouldPGO: false + +steps: +- template: ./checkout.yml + +- powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" + Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" + displayName: 'Extract version numbers' + +- ${{ if eq(parameters.ShouldPGO, 'false') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) + displayName: 'Run build' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- ${{ if eq(parameters.ShouldPGO, 'true') }}: + - powershell: | + $env:SigningCertificate = $null + .\PCbuild\build.bat -v -p $(Platform) --pgo + displayName: 'Run build with PGO' + env: + IncludeUwp: true + Py_OutDir: '$(Build.BinariesDirectory)\bin' + +- powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + displayName: 'Add WinSDK tools to path' + +- powershell: | + $env:SigningCertificate = $null + $(_HostPython) PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch) + makecat "${env:CAT}.cdf" + del "${env:CAT}.cdf" + if (-not (Test-Path "${env:CAT}.cat")) { + throw "Failed to build catalog file" + } + displayName: 'Generate catalog' + env: + CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python + PYTHON_HEXVERSION: $(VersionHex) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries' + condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: bin_$(Name) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish binaries for signing' + condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) + inputs: + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: unsigned_bin_$(Name) + +- task: CopyFiles@2 + displayName: 'Layout Artifact: symbols' + inputs: + sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) + targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) + flatten: true + contents: | + **\*.pdb + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: symbols' + inputs: + PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' + ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml new file mode 100644 index 00000000000000..d42d55fff08dda --- /dev/null +++ b/.azure-pipelines/windows-release/checkout.yml @@ -0,0 +1,21 @@ +parameters: + depth: 3 + +steps: +- checkout: none + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/$(SourceTag))' + condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone (/$(SourceTag))' + condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . + displayName: 'git clone ($(GitRemote)/)' + condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) + +- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . + displayName: 'git clone' + condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml new file mode 100644 index 00000000000000..e4de78555b3f66 --- /dev/null +++ b/.azure-pipelines/windows-release/find-sdk.yml @@ -0,0 +1,17 @@ +# Locate the Windows SDK and add its binaries directory to PATH +# +# `toolname` can be overridden to use a different marker file. + +parameters: + toolname: signtool.exe + +steps: + - powershell: | + $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 + $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) + if (-not $tool) { + throw "SDK is not available" + } + Write-Host "##vso[task.prependpath]$($tool.Directory)" + Write-Host "Adding $($tool.Directory) to PATH" + displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/gpg-sign.yml b/.azure-pipelines/windows-release/gpg-sign.yml new file mode 100644 index 00000000000000..0855af8d703df9 --- /dev/null +++ b/.azure-pipelines/windows-release/gpg-sign.yml @@ -0,0 +1,28 @@ +parameters: + GPGKeyFile: $(GPGKey) + GPGPassphrase: $(GPGPassphrase) + Files: '*' + WorkingDirectory: $(Build.BinariesDirectory) + +steps: +- task: DownloadSecureFile@1 + name: gpgkey + inputs: + secureFile: ${{ parameters.GPGKeyFile }} + displayName: 'Download GPG key' + +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" + gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" + (gci -File ${{ parameters.Files }}).FullName | %{ + gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_ + "Made signature for $_" + } + displayName: 'Generate GPG signatures' + workingDirectory: ${{ parameters.WorkingDirectory }} + +- powershell: | + $p = gps "gpg-agent" -EA 0 + if ($p) { $p.Kill() } + displayName: 'Kill GPG agent' + condition: true diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml new file mode 100644 index 00000000000000..406ccd859faa6a --- /dev/null +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -0,0 +1,23 @@ +steps: +- task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(HostArch)' + condition: and(succeeded(), variables['HostArch']) + inputs: + artifactName: bin_$(HostArch) + targetPath: $(Build.BinariesDirectory)\bin_$(HostArch) + +- powershell: > + Write-Host ( + '##vso[task.setvariable variable=LayoutCmd]& + "$(Python)" + "{1}\PC\layout" + -vv + --source "{1}" + --build "{0}\bin" + --arch "$(Name)" + --temp "{0}\layout-temp" + --include-cat "{0}\bin\python.cat" + --doc-build "{0}\doc"' + -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") + ) + displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml new file mode 100644 index 00000000000000..30f7d34fa61d23 --- /dev/null +++ b/.azure-pipelines/windows-release/mingw-lib.yml @@ -0,0 +1,13 @@ +parameters: + DllToolOpt: -m i386:x86-64 + #DllToolOpt: -m i386 --as-flags=--32 + +steps: +- powershell: | + git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" + gci "bin\$(Arch)\python*.dll" | %{ + & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def + & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} + } + displayName: 'Generate MinGW import library' + workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml new file mode 100644 index 00000000000000..307510a40dd4e5 --- /dev/null +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -0,0 +1,135 @@ +steps: + - template: ./checkout.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: CopyFiles@2 + displayName: 'Merge documentation files' + inputs: + sourceFolder: $(Build.BinariesDirectory)\doc + targetFolder: $(Build.SourcesDirectory)\Doc\build + contents: | + htmlhelp\*.chm + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32' + inputs: + artifactName: bin_win32 + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_win32_d' + inputs: + artifactName: bin_win32_d + targetPath: $(Build.BinariesDirectory)\win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64' + inputs: + artifactName: bin_amd64 + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_amd64_d' + inputs: + artifactName: bin_amd64_d + targetPath: $(Build.BinariesDirectory)\amd64 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_win32' + inputs: + artifactName: tcltk_lib_win32 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_amd64' + inputs: + artifactName: tcltk_lib_amd64 + targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 + + - powershell: | + copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - script: | + call Tools\msi\get_externals.bat + call PCbuild\find_python.bat + echo ##vso[task.setvariable variable=PYTHON]%PYTHON% + call PCbuild/find_msbuild.bat + echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% + displayName: 'Get external dependencies' + + - script: | + %PYTHON% -m pip install blurb + %PYTHON% -m blurb merge -f Misc\NEWS + displayName: 'Merge NEWS file' + + - script: | + %MSBUILD% Tools\msi\launcher\launcher.wixproj + displayName: 'Build launcher installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build win32 installer' + env: + Platform: x86 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\win32\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 + BuildForRelease: true + SuppressMinGWLib: true + + - script: | + %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true + %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false + displayName: 'Build amd64 installer' + env: + Platform: x64 + Py_OutDir: $(Build.BinariesDirectory) + PYTHON: $(Build.BinariesDirectory)\amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 + BuildForRelease: true + SuppressMinGWLib: true + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (1/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\win32\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 + contents: | + *.msi + *.cab + *.exe + + - task: CopyFiles@2 + displayName: 'Assemble artifact: msi (2/2)' + inputs: + sourceFolder: $(Build.BinariesDirectory)\amd64\en-us + targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 + contents: | + *.msi + *.cab + *.exe + + - task: PublishPipelineArtifact@0 + displayName: 'Publish MSI' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\msi' + artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml new file mode 100644 index 00000000000000..69f3b1e16451ec --- /dev/null +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -0,0 +1,179 @@ +jobs: +- job: Build_Docs + displayName: Docs build + pool: + name: 'Windows Release' + #vmImage: windows-2019 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: Doc\make.bat html + displayName: 'Build HTML docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + - script: Doc\make.bat htmlhelp + displayName: 'Build CHM docs' + env: + BUILDDIR: $(Build.BinariesDirectory)\Doc + + #- powershell: | + # mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp" + # iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm" + # displayName: 'Cheat at building CHM docs' + + - task: CopyFiles@2 + displayName: 'Assemble artifact: Doc' + inputs: + sourceFolder: $(Build.BinariesDirectory)\Doc + targetFolder: $(Build.ArtifactStagingDirectory)\Doc + contents: | + html\**\* + htmlhelp\*.chm + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: doc' + inputs: + targetPath: $(Build.ArtifactStagingDirectory)\Doc + artifactName: doc + +- job: Build_Python + displayName: Python build + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Arch: win32 + Platform: x86 + Configuration: Release + _HostPython: .\python + win32_d: + Name: win32_d + Arch: win32 + Platform: x86 + Configuration: Debug + _HostPython: .\python + amd64_d: + Name: amd64_d + Arch: amd64 + Platform: x64 + Configuration: Debug + _HostPython: .\python + arm64: + Name: arm64 + Arch: arm64 + Platform: ARM64 + Configuration: Release + _HostPython: python + arm64_d: + Name: arm64_d + Arch: arm64 + Platform: ARM64 + Configuration: Debug + _HostPython: python + + steps: + - template: ./build-steps.yml + +- job: Build_Python_NonPGO + displayName: Python non-PGO build + condition: and(succeeded(), ne(variables['DoPGO'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + _HostPython: .\python + + steps: + - template: ./build-steps.yml + + +- job: Build_Python_PGO + displayName: Python PGO build + condition: and(succeeded(), eq(variables['DoPGO'], 'true')) + + # Allow up to five hours for PGO + timeoutInMinutes: 300 + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Arch: amd64 + Platform: x64 + Configuration: Release + _HostPython: .\python + + steps: + - template: ./build-steps.yml + parameters: + ShouldPGO: true + + +- job: TclTk_Lib + displayName: Publish Tcl/Tk Library + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - script: PCbuild\get_externals.bat --no-openssl --no-libffi + displayName: 'Get external dependencies' + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x86 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" + + - task: MSBuild@1 + displayName: 'Copy Tcl/Tk lib for publish' + inputs: + solution: PCbuild\tcltk.props + platform: x64 + msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_win32' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' + artifactName: tcltk_lib_win32 + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: tcltk_lib_amd64' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + artifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml new file mode 100644 index 00000000000000..dbccdead143b21 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -0,0 +1,61 @@ +jobs: +- job: Make_Embed_Layout + displayName: Make embeddable layout + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - template: ./layout-command.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + displayName: 'Extract version numbers' + + - powershell: > + $(LayoutCmd) + --copy "$(Build.ArtifactStagingDirectory)\layout" + --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" + --preset-embed + displayName: 'Generate embeddable layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_embed_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_embed_$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: embed' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' + ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml new file mode 100644 index 00000000000000..8fc8da3e52fe03 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -0,0 +1,75 @@ +jobs: +- job: Make_Layouts + displayName: Make layouts + condition: and(succeeded(), eq(variables['DoLayout'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: doc' + inputs: + artifactName: doc + targetPath: $(Build.BinariesDirectory)\doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default + displayName: 'Generate full layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_full_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml new file mode 100644 index 00000000000000..def4f7d3c6bee5 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -0,0 +1,97 @@ +jobs: +- job: Make_MSIX_Layout + displayName: Make MSIX layout + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + #win32: + # Name: win32 + # Python: $(Build.BinariesDirectory)\bin\python.exe + # PYTHONHOME: $(Build.SourcesDirectory) + # TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)_d' + inputs: + artifactName: bin_$(Name)_d + targetPath: $(Build.BinariesDirectory)\bin + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: tcltk_lib_$(Name)' + condition: and(succeeded(), variables['TclLibrary']) + inputs: + artifactName: tcltk_lib_$(Name) + targetPath: $(Build.BinariesDirectory)\tcltk_lib + + - powershell: | + copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile + displayName: 'Generate store APPX layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appxstore_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' + artifactName: layout_appxstore_$(Name) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: cert' + condition: and(succeeded(), variables['SigningCertificate']) + inputs: + artifactName: cert + targetPath: $(Build.BinariesDirectory)\cert + + - powershell: | + $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) + Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" + Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" + Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" + displayName: 'Override signing parameters' + condition: and(succeeded(), variables['SigningCertificate']) + + - powershell: | + Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests + displayName: 'Generate sideloading APPX layout' + env: + TCL_LIBRARY: $(TclLibrary) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_appx_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\appx' + artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml new file mode 100644 index 00000000000000..41cdff850e83be --- /dev/null +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -0,0 +1,52 @@ +jobs: +- job: Make_Nuget_Layout + displayName: Make Nuget layout + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + amd64: + Name: amd64 + Python: $(Build.BinariesDirectory)\bin\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + arm64: + Name: arm64 + HostArch: amd64 + Python: $(Build.BinariesDirectory)\bin_amd64\python.exe + PYTHONHOME: $(Build.SourcesDirectory) + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: bin_$(Name)' + inputs: + artifactName: bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force + displayName: 'Copy signed files into sources' + condition: and(succeeded(), variables['SigningCertificate']) + + - template: ./layout-command.yml + + - powershell: | + $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget + displayName: 'Generate nuget layout' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: layout_nuget_$(Name)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\nuget' + artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml new file mode 100644 index 00000000000000..9b965b09c14748 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-msi.yml @@ -0,0 +1,36 @@ +jobs: +- job: Make_MSI + displayName: Make MSI + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmImage: windows-2019 + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml + +- job: Make_Signed_MSI + displayName: Make signed MSI + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + variables: + ReleaseUri: http://www.python.org/{arch} + DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} + Py_OutDir: $(Build.BinariesDirectory) + + workspace: + clean: all + + steps: + - template: msi-steps.yml diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml new file mode 100644 index 00000000000000..f967cfdbe326f8 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -0,0 +1,144 @@ +jobs: +- job: Pack_MSIX + displayName: Pack MSIX bundles + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + Artifact: appx + Suffix: + ShouldSign: true + amd64_store: + Name: amd64 + Artifact: appxstore + Suffix: -store + Upload: true + arm64: + Name: arm64 + Artifact: appx + Suffix: + ShouldSign: true + arm64_store: + Name: arm64 + Artifact: appxstore + Suffix: -store + Upload: true + + steps: + - template: ./checkout.yml + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_$(Artifact)_$(Name)' + inputs: + artifactName: layout_$(Artifact)_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: symbols' + inputs: + artifactName: symbols + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" + Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" + Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" + Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" + Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" + displayName: 'Extract version numbers' + + - powershell: | + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + displayName: 'Build msix' + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb + displayName: 'Build appxsym' + workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' + ArtifactName: unsigned_msix + + - powershell: | + 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * + displayName: 'Build msixupload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + workingDirectory: $(Build.ArtifactStagingDirectory)\msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIXUpload' + condition: and(succeeded(), eq(variables['Upload'], 'true')) + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' + ArtifactName: msixupload + + +- job: Sign_MSIX + displayName: Sign side-loadable MSIX bundles + dependsOn: + - Pack_MSIX + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + - template: ./find-sdk.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Artifact: unsigned_msix' + inputs: + artifactName: unsigned_msix + downloadPath: $(Build.BinariesDirectory) + + # MSIX must be signed and timestamped simultaneously + - powershell: | + $failed = $true + foreach ($retry in 1..3) { + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix) + if ($?) { + $failed = $false + break + } + sleep 1 + } + if ($failed) { + throw "Failed to sign MSIX" + } + displayName: 'Sign MSIX' + workingDirectory: $(Build.BinariesDirectory)\unsigned_msix + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: MSIX' + inputs: + PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' + ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml new file mode 100644 index 00000000000000..8dfea382c35622 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -0,0 +1,51 @@ +jobs: +- job: Pack_Nuget + displayName: Pack Nuget bundles + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + amd64: + Name: amd64 + win32: + Name: win32 + arm64: + Name: arm64 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: layout_nuget_$(Name)' + inputs: + artifactName: layout_nuget_$(Name) + targetPath: $(Build.BinariesDirectory)\layout + + - task: NugetToolInstaller@0 + displayName: 'Install Nuget' + inputs: + versionSpec: '>=5.0' + + - powershell: | + nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + displayName: 'Create nuget package' + + - powershell: | + gci *.nupkg | %{ + nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite + } + displayName: 'Sign nuget package' + workingDirectory: $(Build.ArtifactStagingDirectory) + condition: and(succeeded(), variables['SigningCertificate']) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: nuget' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml new file mode 100644 index 00000000000000..d5edf44ef5c2ec --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml @@ -0,0 +1,46 @@ +jobs: +- job: Publish_Nuget + displayName: Publish Nuget packages + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\nuget' + condition: and(succeeded(), not(variables['PublishArmPackages'])) + + - task: NuGetCommand@2 + displayName: Push packages + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + inputs: + command: push + packagesToPush: '$(Build.BinariesDirectory)\nuget\*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml new file mode 100644 index 00000000000000..b4710d8c248e23 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml @@ -0,0 +1,158 @@ +jobs: +- job: Publish_Python + displayName: Publish python.org packages + condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'))) + + pool: + #vmImage: windows-2019 + name: 'Windows Release' + + workspace: + clean: all + + steps: + - template: ./checkout.yml + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: Doc' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' + displayName: 'Prevent publishing ARM/ARM64 packages' + workingDirectory: '$(Build.BinariesDirectory)\embed' + condition: and(succeeded(), not(variables['PublishArmPackages'])) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): Doc' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: Doc + targetPath: $(Build.BinariesDirectory)\Doc + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact from $(BuildToPublish): msi' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + buildType: specific + project: cpython + pipeline: 21 + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact from $(BuildToPublish): embed' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + + - template: ./gpg-sign.yml + parameters: + GPGKeyFile: 'python-signing.key' + Files: 'doc\htmlhelp\*.chm, msi\*\*, embed\*.zip' + + - powershell: > + $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 + -build msi + -user $(PyDotOrgUsername) + -server $(PyDotOrgServer) + -doc_htmlhelp doc\htmlhelp + -embed embed + -skippurge + -skiptest + -skiphash + condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Upload files to python.org' + + - powershell: > + python + "$(Build.SourcesDirectory)\Tools\msi\purge.py" + (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Purge CDN' + + - powershell: | + $failures = 0 + gci "msi\*\*-webinstall.exe" -File | %{ + $d = mkdir "tests\$($_.BaseName)" -Force + gci $d -r -File | del + $ic = copy $_ $d -PassThru + "Checking layout for $($ic.Name)" + Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" + if (-not $?) { + Write-Error "Failed to validate layout of $($inst.Name)" + $failures += 1 + } + } + if ($failures) { + Write-Error "Failed to validate $failures installers" + exit 1 + } + #condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Test layouts' + + - powershell: | + $hashes = gci doc\htmlhelp\python*.chm, msi\*\*.exe, embed\*.zip | ` + Sort-Object Name | ` + Format-Table Name, @{ + Label="MD5"; + Expression={(Get-FileHash $_ -Algorithm MD5).Hash} + }, Length -AutoSize | ` + Out-String -Width 4096 + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + $hashes | Out-File "$d\hashes.txt" -Encoding ascii + $hashes + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Generate hashes' + + - powershell: | + "Copying:" + (gci msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc).FullName + $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force + move msi\*\python*.asc, doc\htmlhelp\*.asc, embed\*.asc $d -Force + gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } + workingDirectory: $(Build.BinariesDirectory) + displayName: 'Copy GPG signatures for build' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: hashes' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)\hashes' + artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml new file mode 100644 index 00000000000000..e0512b95f27da8 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-publish-store.yml @@ -0,0 +1,35 @@ +jobs: +- job: Publish_Store + displayName: Publish Store packages + condition: and(succeeded(), eq(variables['DoMSIX'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), not(variables['BuildToPublish'])) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: msixupload' + condition: and(succeeded(), variables['BuildToPublish']) + inputs: + artifactName: msixupload + downloadPath: $(Build.BinariesDirectory) + buildType: specific + project: cpython + pipeline: Windows-Release + buildVersionToDownload: specific + buildId: $(BuildToPublish) + + # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) + # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml new file mode 100644 index 00000000000000..c21e1c9f2b0f9b --- /dev/null +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -0,0 +1,130 @@ +parameters: + Include: '*.exe, *.dll, *.pyd, *.cat, *.ps1' + Exclude: 'vcruntime*, libffi*, libcrypto*, libssl*' + +jobs: +- job: Sign_Python + displayName: Sign Python binaries + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + arm64: + Name: arm64 + + steps: + - template: ./checkout.yml + - template: ./find-sdk.yml + + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + + - powershell: | + Write-Host "##vso[build.addbuildtag]signed" + displayName: 'Add build tags' + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: unsigned_bin_$(Name)' + inputs: + artifactName: unsigned_bin_$(Name) + targetPath: $(Build.BinariesDirectory)\bin + + - powershell: | + copy "$(Build.SourcesDirectory)\Lib\venv\scripts\common\Activate.ps1" . + displayName: 'Copy files from source' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files + displayName: 'Sign binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + + - powershell: | + $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) + $failed = $true + foreach ($retry in 1..10) { + signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files + if ($?) { + $failed = $false + break + } + sleep 5 + } + if ($failed) { + Write-Host "##vso[task.logissue type=error]Failed to timestamp files" + } + displayName: 'Timestamp binaries' + workingDirectory: $(Build.BinariesDirectory)\bin + continueOnError: true + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: bin_$(Name)' + inputs: + targetPath: '$(Build.BinariesDirectory)\bin' + artifactName: bin_$(Name) + + +- job: Dump_CertInfo + displayName: Capture certificate info + condition: and(succeeded(), variables['SigningCertificate']) + + pool: + name: 'Windows Release' + + steps: + - checkout: none + + - powershell: | + $m = 'CN=$(SigningCertificate)' + $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` + ?{ $_.Subject -match $m } | ` + select -First 1 + if (-not $c) { + Write-Host "Failed to find certificate for $(SigningCertificate)" + exit + } + $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force + $cf = "$d\cert.cer" + [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) + $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value + + $info = @{ Subject=$c.Subject; SHA256=$csha; } + + $d = mkdir "$(Build.BinariesDirectory)\cert" -Force + $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" + displayName: "Extract certificate info" + + - task: PublishPipelineArtifact@0 + displayName: 'Publish artifact: cert' + inputs: + targetPath: '$(Build.BinariesDirectory)\cert' + artifactName: cert + + +- job: Mark_Unsigned + displayName: Tag unsigned build + condition: and(succeeded(), not(variables['SigningCertificate'])) + + pool: + vmImage: windows-2019 + + steps: + - checkout: none + + - powershell: | + Write-Host "##vso[build.addbuildtag]unsigned" + displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml new file mode 100644 index 00000000000000..d99bd74722bacb --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-embed.yml @@ -0,0 +1,41 @@ +jobs: +- job: Test_Embed + displayName: Test Embed + condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Name: win32 + amd64: + Name: amd64 + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: embed' + inputs: + artifactName: embed + downloadPath: $(Build.BinariesDirectory) + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" + Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Install Python and add to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml new file mode 100644 index 00000000000000..21e38c39590f70 --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -0,0 +1,108 @@ +jobs: +- job: Test_MSI + displayName: Test MSI + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32_User: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User + InstallAllUsers: 0 + win32_Machine: + ExeMatch: 'python-[\dabrc.]+-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine + InstallAllUsers: 1 + amd64_User: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User + InstallAllUsers: 0 + amd64_Machine: + ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe' + Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine + InstallAllUsers: 1 + + steps: + - checkout: none + + - task: DownloadPipelineArtifact@1 + displayName: 'Download artifact: msi' + inputs: + artifactName: msi + targetPath: $(Build.BinariesDirectory)\msi + + - powershell: | + $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) + Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" + Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" + displayName: 'Find installer executable' + workingDirectory: $(Build.BinariesDirectory)\msi + + - script: > + "$(SetupExe)" + /passive + /log "$(Logs)\install\log.txt" + TargetDir="$(Build.BinariesDirectory)\Python" + Include_debug=1 + Include_symbols=1 + InstallAllUsers=$(InstallAllUsers) + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\Python\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - powershell: | + gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-machine Start Menu items' + - powershell: | + gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" + displayName: 'Capture per-user Start Menu items' + + - powershell: | + gci -r "HKLM:\Software\WOW6432Node\Python" + displayName: 'Capture per-machine 32-bit registry' + - powershell: | + gci -r "HKLM:\Software\Python" + displayName: 'Capture per-machine native registry' + - powershell: | + gci -r "HKCU:\Software\Python" + displayName: 'Capture current-user registry' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' + + - script: | + python -m test -uall -v test_ttk_guionly test_tk test_idle + displayName: 'Test Tkinter and Idle' + + - script: > + "$(SetupExe)" + /passive + /uninstall + /log "$(Logs)\uninstall\log.txt" + displayName: 'Uninstall Python' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: logs' + condition: true + continueOnError: true + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' + ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml new file mode 100644 index 00000000000000..94d815e95226ef --- /dev/null +++ b/.azure-pipelines/windows-release/stage-test-nuget.yml @@ -0,0 +1,58 @@ +jobs: +- job: Test_Nuget + displayName: Test Nuget + condition: and(succeeded(), eq(variables['DoNuget'], 'true')) + + pool: + vmImage: windows-2019 + + workspace: + clean: all + + strategy: + matrix: + win32: + Package: pythonx86 + amd64: + Package: python + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download artifact: nuget' + inputs: + artifactName: nuget + downloadPath: $(Build.BinariesDirectory) + + - task: NugetToolInstaller@0 + inputs: + versionSpec: '>= 5' + + - powershell: > + nuget install + $(Package) + -Source "$(Build.BinariesDirectory)\nuget" + -OutputDirectory "$(Build.BinariesDirectory)\install" + -Prerelease + -ExcludeVersion + -NonInteractive + displayName: 'Install Python' + + - powershell: | + $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" + Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" + displayName: 'Add test Python to PATH' + + - script: | + python -c "import sys; print(sys.version)" + displayName: 'Collect version number' + + - script: | + python -m site + displayName: 'Collect site' + + - script: | + python -m pip install "azure<0.10" + python -m pip uninstall -y azure python-dateutil six + displayName: 'Test (un)install package' diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml index 794a23a5d77e86..f502c40637c310 100644 --- a/.azure-pipelines/windows-steps.yml +++ b/.azure-pipelines/windows-steps.yml @@ -19,9 +19,11 @@ steps: - script: python.bat -m test.pythoninfo displayName: 'Display build info' + condition: and(succeeded(), variables['testRunPlatform']) - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test" displayName: 'Tests' + condition: and(succeeded(), variables['testRunPlatform']) env: PREFIX: $(Py_OutDir)\$(arch) @@ -32,4 +34,4 @@ steps: mergeTestResults: true testRunTitle: $(testRunTitle) platform: $(testRunPlatform) - condition: succeededOrFailed() + condition: and(succeededOrFailed(), variables['testRunPlatform']) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000000..81445d2d79c739 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*.{py,c,cpp,h,rst,md,yml}] +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space + +[*.{py,c,cpp,h}] +indent_size = 4 + +[*.yml] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes index c9a54fbd472ecc..bec16a08152ebb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -54,7 +54,7 @@ Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true Python/opcode_targets.h linguist-generated=true Objects/typeslots.inc linguist-generated=true -Modules/unicodedata_db.h linguist-generated=true +*_db.h linguist-generated=true Doc/library/token-list.inc linguist-generated=true Include/token.h linguist-generated=true Lib/token.py linguist-generated=true diff --git a/.github/appveyor.yml b/.github/appveyor.yml deleted file mode 100644 index e8012f69ee5b2e..00000000000000 --- a/.github/appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 3.8build{build} -clone_depth: 5 -branches: - only: - - master - - /\d\.\d/ - - buildbot-custom -cache: - - externals -> PCbuild -before_build: - - ps: |+ - if ($env:APPVEYOR_RE_BUILD) { - echo 'Doing full build due to re-build request.' - } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { - echo 'Not a PR, doing full build.' - } else { - git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH - $mergebase = git merge-base HEAD FETCH_HEAD - $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)' - If (!$changes) { - echo 'Only docs were updated, stopping build process.' - Exit-AppveyorBuild - } else { - echo 'Doing full build due to non-doc changes in these files:' - echo $changes - } - } - - -build_script: - - cmd: PCbuild\build.bat -e -p x64 - - cmd: PCbuild\amd64\python.exe -m test.pythoninfo -test_script: - - cmd: PCbuild\rt.bat -x64 -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 --fail-env-changed -j0 -environment: - HOST_PYTHON: C:\Python36\python.exe -image: - - Visual Studio 2017 diff --git a/.github/codecov.yml b/.github/codecov.yml index 9d97dfbc43f8d0..ea504f48672eac 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -5,7 +5,7 @@ codecov: comment: off ignore: - "Doc/**/*" - - "Misc/*" + - "Misc/**/*" - "Mac/**/*" - "PC/**/*" - "PCbuild/**/*" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000000..700fe10e034fec --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,251 @@ +name: Tests + +# bpo-40548: "paths-ignore" is not used to skip documentation-only PRs, because +# it prevents to mark a job as mandatory. A PR cannot be merged if a job is +# mandatory but not scheduled because of "paths-ignore". +on: + pull_request: + branches: + - master + - 3.8 + - 3.7 + +jobs: + check_source: + name: 'Check for source changes' + runs-on: ubuntu-latest + outputs: + run_tests: ${{ steps.check.outputs.run_tests }} + run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }} + steps: + - uses: actions/checkout@v4 + - name: Check for source changes + id: check + run: | + if [ -z "$GITHUB_BASE_REF" ]; then + echo '::set-output name=run_tests::true' + echo '::set-output name=run_ssl_tests::true' + else + git fetch origin $GITHUB_BASE_REF --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true + fi + + check_abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + run: make check-abidump + + check_generated_files: + name: 'Check if generated files are up to date' + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Check Autoconf version 2.69 and aclocal 1.16.3 + run: | + grep "Generated by GNU Autoconf 2.69" configure + grep "aclocal 1.16.3" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Configure CPython + run: | + # Build Python with the libpython dynamic library + ./configure --config-cache --with-pydebug --enable-shared + - name: Regenerate autoconf files + run: | + make regen-configure + - name: Build CPython + run: | + ./configure --with-pydebug + make -j4 regen-all + - name: Check for changes + run: | + changes=$(git status --porcelain) + # Check for changes in regenerated files + if ! test -z "$changes" + then + echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)" + echo "$changes" + exit 1 + fi + - name: Check exported libpython symbols + run: make smelly + + build_win32: + name: 'Windows (x86)' + runs-on: windows-2019 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Build CPython + run: .\PCbuild\build.bat -e -p Win32 + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -p Win32 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + build_win_amd64: + name: 'Windows (x64)' + runs-on: windows-2019 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v4 + - name: Build CPython + run: .\PCbuild\build.bat -e -p x64 + - name: Display build info + run: .\python.bat -m test.pythoninfo + - name: Tests + run: .\PCbuild\rt.bat -p x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + + build_macos: + name: 'macOS' + runs-on: macos-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + steps: + - uses: actions/checkout@v4 + - name: Install Homebrew dependencies + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + - name: Configure CPython + run: | + SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.sdk \ + CC=clang \ + CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ + LDFLAGS="-L$(brew --prefix gdbm)/lib -L$(brew --prefix xz)/lib" \ + ./configure --prefix=/opt/python-dev \ + --with-pydebug \ + --with-openssl="$(brew --prefix openssl@3.0)" \ + --with-tcltk-libs="$(pkg-config --libs tk)" \ + --with-tcltk-includes="$(pkg-config --cflags tk)" + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu: + name: 'Ubuntu' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 1.1.1v + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v2.1.3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu_ssltests: + name: 'Ubuntu SSL tests with OpenSSL' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_ssl_tests == 'true' + strategy: + fail-fast: false + matrix: + openssl_ver: [1.0.2u, 1.1.1v, 3.0.10, 3.1.2] + env: + OPENSSL_VER: ${{ matrix.openssl_ver }} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: SSL tests + run: ./python Lib/test/ssltests.py diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml new file mode 100644 index 00000000000000..f9400b9ea8ac26 --- /dev/null +++ b/.github/workflows/build_msi.yml @@ -0,0 +1,36 @@ +name: TestsMSI + +on: + push: + branches: + - master + - 3.8 + - 3.7 + paths: + - 'Tools/msi/**' + - 'Lib/ensurepip/**' + pull_request: + branches: + - master + - 3.8 + - 3.7 + paths: + - 'Tools/msi/**' + - 'Lib/ensurepip/**' + +jobs: + build_win32: + name: 'Windows (x86) Installer' + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Build CPython installer + run: .\Tools\msi\build.bat -x86 + + build_win_amd64: + name: 'Windows (x64) Installer' + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Build CPython installer + run: .\Tools\msi\build.bat -x64 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000000..4ed1e7d4ebab8d --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,89 @@ +name: Coverage + +on: + push: + branches: + - master + - 3.8 + - 3.7 + paths-ignore: + - 'Doc/**' + - 'Misc/**' + #pull_request: + # branches: + # - master + # - 3.8 + # - 3.7 + # paths-ignore: + # - 'Doc/**' + # - 'Misc/**' + +jobs: + coverage_ubuntu: + name: 'Ubuntu (Coverage)' + runs-on: ubuntu-latest + env: + OPENSSL_VER: 1.1.1n + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v2.1.3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + - name: Configure CPython + run: ./configure --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: 'Coverage Preparation' + run: | + ./python -m venv .venv + source ./.venv/bin/activate + python -m pip install -U coverage + python -m test.pythoninfo + - name: 'Tests with coverage' + run: > + source ./.venv/bin/activate && + xvfb-run python -m coverage + run --branch --pylib + -m test + --fail-env-changed + -uall,-cpu + -x test_multiprocessing_fork + -x test_multiprocessing_forkserver + -x test_multiprocessing_spawn + -x test_concurrent_futures + || true + - name: 'Publish code coverage results' + run: | + source ./.venv/bin/activate + bash <(curl -s https://codecov.io/bash) + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + c_coverage_ubuntu: + name: 'Ubuntu (C Coverage)' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure CPython + run: ./configure + - name: 'Build CPython and measure coverage' + run: xvfb-run make -j4 coverage-report + - name: 'Publish code coverage results' + if: always() + run: | + make pythoninfo + bash <(curl -s https://codecov.io/bash) + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 00000000000000..51612a748634e3 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,40 @@ +name: Docs + +on: + #push: + # branches: + # - master + # - 3.8 + # - 3.7 + # paths: + # - 'Doc/**' + pull_request: + branches: + - master + - 3.8 + - 3.7 + paths: + - 'Doc/**' + - 'Misc/**' + +jobs: + build_doc: + name: 'Docs' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: 'Install Dependencies' + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + - name: 'Configure CPython' + run: ./configure --with-pydebug + - name: 'Build CPython' + run: make -j4 + - name: 'Install build dependencies' + run: make -C Doc/ PYTHON=../python venv + - name: 'Build documentation' + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest suspicious html + - name: 'Upload' + uses: actions/upload-artifact@v2.2.2 + with: + name: doc-html + path: Doc/build/html diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh new file mode 100755 index 00000000000000..56cc70edf60001 --- /dev/null +++ b/.github/workflows/posix-deps-apt.sh @@ -0,0 +1,22 @@ +#!/bin/sh +apt-get update + +apt-get -yq install \ + build-essential \ + ccache \ + gdb \ + lcov \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + lzma \ + lzma-dev \ + tk-dev \ + uuid-dev \ + xvfb \ + zlib1g-dev diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml new file mode 100644 index 00000000000000..9a36240146964c --- /dev/null +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -0,0 +1,28 @@ +name: Verify bundled pip and setuptools + +on: + workflow_dispatch: + push: + paths: + - 'Lib/ensurepip/_bundled/**' + - '.github/workflows/verify-ensurepip-wheels.yml' + - 'Tools/scripts/verify_ensurepip_wheels.py' + pull_request: + paths: + - 'Lib/ensurepip/_bundled/**' + - '.github/workflows/verify-ensurepip-wheels.yml' + - 'Tools/scripts/verify_ensurepip_wheels.py' + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3' + - name: Compare checksums of bundled pip and setuptools to ones published on PyPI + run: ./Tools/scripts/verify_ensurepip_wheels.py diff --git a/.gitignore b/.gitignore index 9445ef1e2c5252..24d5a244e0e0bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,14 @@ -# added for local development -.buildaix/ -Modules/python.exp -buildaix/ -installp/ -.gitignore +##### +# First, rules intended to apply in all subdirectories. +# These contain no slash, or only a trailing slash. -# Two-trick pony for OSX and other case insensitive file systems: -# Ignore ./python binary on Unix but still look into ./Python/ directory. -/python -!/Python/ *.cover *.iml *.o +*.a +*.so* +*.dylib +*.dll *.orig *.pyc *.pyd @@ -24,6 +21,31 @@ installp/ *.profraw *.dyn .gdb_history +.purify +__pycache__ +.hg/ +.svn/ +.idea/ +tags +TAGS +.vs/ +.vscode/ +gmon.out +.coverage +.mypy_cache/ + +*.exe +!Lib/distutils/command/*.exe + +# Ignore core dumps... but not Tools/msi/core/ or the like. +core +!core/ + + +##### +# Then, rules meant for a specific location relative to the repo root. +# These must contain a non-trailing slash (and may also have a trailing slash.) + Doc/build/ Doc/venv/ Doc/.venv/ @@ -33,23 +55,20 @@ Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* -Makefile -Makefile.pre +!Lib/test/data/README +/Makefile +/Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh -Modules/Setup Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix Programs/_freeze_importlib -Programs/_freeze_importlib.exe Programs/_testembed -Programs/_testembed.exe PC/python_nt*.h PC/pythonnt_rc*.h -PC/*/*.exe PC/*/*.exp PC/*/*.lib PC/*/*.bsc @@ -68,51 +87,44 @@ PCbuild/*-pgi PCbuild/*-pgo PCbuild/*.VC.db PCbuild/*.VC.opendb -PCbuild/.vs/ PCbuild/amd64/ PCbuild/arm32/ PCbuild/arm64/ PCbuild/obj/ PCbuild/win32/ -.purify -__pycache__ -autom4te.cache -build/ -buildno -config.cache -config.log -config.status -config.status.lineno -core -db_home -.hg/ -.idea/ -ipch/ -libpython*.a -libpython*.so* -libpython*.dylib -libpython*.dll -platform -pybuilddir.txt -pyconfig.h -python-config -python-config.py -python.bat -python.exe -python-gdb.py -python.exe-gdb.py -reflog.txt -.svn/ -tags -TAGS -.coverage -coverage/ -externals/ -htmlcov/ +/autom4te.cache +/build/ +/config.cache +/config.log +/config.status +/config.status.lineno +/platform +/profile-clean-stamp +/profile-run-stamp +/pybuilddir.txt +/pyconfig.h +/python-config +/python-config.py +/python.bat +/python-gdb.py +/python.exe-gdb.py +/reflog.txt +/coverage/ +/externals/ +/htmlcov/ Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 -.vs/ -.vscode/ -gmon.out -.mypy_cache/ + +# Two-trick pony for OSX and other case insensitive file systems: +# Ignore ./python binary on Unix but still look into ./Python/ directory. +/python +!/Python/ + +# Artifacts generated by 3.11 lying around when switching branches: +/_bootstrap_python +/Programs/_freeze_module +/Modules/Setup.bootstrap +/Modules/Setup.stdlib +/Python/deepfreeze/ +/Python/frozen_modules/ \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000000..6a9db718698269 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +# This is a dummy config file so that readthedocs.org doesn't fail on security branches. +# Note that this won't result in docs actually getting built; +# clicking on the docs preview link on a PR will result in a 404. +version: 2 +formats: [] +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + jobs: + post_checkout: + - exit 183 diff --git a/.travis.yml b/.travis.yml index 02de997750abc8..f347b258d50889 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: c dist: xenial -group: beta # To cache doc-building dependencies and C compiler output. cache: @@ -11,11 +10,10 @@ cache: env: global: - - OPENSSL=1.1.1c + - OPENSSL=1.1.1k - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - # Use -O3 because we don't use debugger on Travis-CI - - CFLAGS="-I${OPENSSL_DIR}/include -O3" + - CFLAGS="-I${OPENSSL_DIR}/include" - LDFLAGS="-L${OPENSSL_DIR}/lib" # Set rpath with env var instead of -Wl,-rpath linker flag # OpenSSL ignores LDFLAGS when linking bin/openssl @@ -43,6 +41,7 @@ matrix: addons: apt: packages: + - gdb - xvfb - name: "Documentation build" os: linux @@ -52,10 +51,7 @@ matrix: env: TESTING=docs before_script: - cd Doc - # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. - # (Updating the version is fine as long as no warnings are raised by doing so.) - # The theme used by the docs is stored separately, so we need to install that as well. - - python -m pip install sphinx==1.8.2 blurb python-docs-theme + - make venv PYTHON=python script: - make check suspicious html SPHINXOPTS="-q -W -j4" - name: "Documentation tests" @@ -73,16 +69,6 @@ matrix: - make -C Doc/ PYTHON=../python venv script: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest - - name: "Mac OS X tests" - os: osx - language: c - compiler: clang - # Testing under macOS is optional until testing stability has been demonstrated. - env: OPTIONAL=true - before_install: - # Python 3 is needed for Argument Clinic and multissl - - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3 - - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH - name: "Test code coverage (Python)" os: linux language: c @@ -93,6 +79,12 @@ matrix: packages: - xvfb before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run Python coverage on pull requests." + exit + fi - ./configure - make -j4 # Need a venv that can parse covered code. @@ -117,6 +109,12 @@ matrix: - lcov - xvfb before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run C coverage on pull requests." + exit + fi - ./configure script: - xvfb-run make -j4 coverage-report @@ -166,7 +164,8 @@ install: # Travis provides only 2 cores, so don't overdo the parallelism and waste memory. before_script: - - ./configure --with-pydebug + # -Og is much faster than -O0 + - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug - make -j4 regen-all - changes=`git status --porcelain` - | diff --git a/Doc/Makefile b/Doc/Makefile index 6f86728ea834c3..6bf1f408b56f0d 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,10 +13,14 @@ SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) SPHINXERRORHANDLING = -W -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \ +# Internal variables. +PAPEROPT_a4 = -D latex_elements.papersize=a4paper +PAPEROPT_letter = -D latex_elements.papersize=letterpaper + +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) -.PHONY: help build html htmlhelp latex text changes linkcheck \ +.PHONY: help build html htmlhelp latex text texinfo changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ autobuild-dev autobuild-stable venv @@ -29,6 +33,7 @@ help: @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" + @echo " texinfo to make Texinfo file" @echo " epub to make EPUB files" @echo " changes to make an overview over all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @@ -85,6 +90,11 @@ text: BUILDER = text text: build @echo "Build finished; the text files are in build/text." +texinfo: BUILDER = texinfo +texinfo: build + @echo "Build finished; the python.texi file is in build/texinfo." + @echo "Run \`make info' in that directory to run it through makeinfo." + epub: BUILDER = epub epub: build @echo "Build finished; the epub files are in build/epub." @@ -133,7 +143,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) $(VENVDIR)/bin/python3 -m pip install -U pip setuptools - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt @echo "The venv has been created in the $(VENVDIR) directory" dist: @@ -179,6 +189,17 @@ dist: make epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + # archive the texinfo build + rm -rf build/texinfo + make texinfo + make info --directory=build/texinfo + cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo + tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) + rm -r dist/python-$(DISTVERSION)-docs-texinfo + rm dist/python-$(DISTVERSION)-docs-texinfo.tar + check: $(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst @@ -194,12 +215,12 @@ serve: # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 1e044ad2033d87..e1c2d6a2a105d6 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -40,38 +40,39 @@ though it may take a while to be processed. Using the Python issue tracker ============================== -Bug reports for Python itself should be submitted via the Python Bug Tracker -(https://bugs.python.org/). The bug tracker offers a Web form which allows -pertinent information to be entered and submitted to the developers. +Issue reports for Python itself should be submitted via the GitHub issues +tracker (https://github.com/python/cpython/issues). +The GitHub issues tracker offers a web form which allows pertinent information +to be entered and submitted to the developers. The first step in filing a report is to determine whether the problem has already been reported. The advantage in doing so, aside from saving the -developers time, is that you learn what has been done to fix it; it may be that +developers' time, is that you learn what has been done to fix it; it may be that the problem has already been fixed for the next release, or additional information is needed (in which case you are welcome to provide it if you can!). -To do this, search the bug database using the search box on the top of the page. +To do this, search the tracker using the search box at the top of the page. -If the problem you're reporting is not already in the bug tracker, go back to -the Python Bug Tracker and log in. If you don't already have a tracker account, -select the "Register" link or, if you use OpenID, one of the OpenID provider -logos in the sidebar. It is not possible to submit a bug report anonymously. +If the problem you're reporting is not already in the list, log in to GitHub. +If you don't already have a GitHub account, create a new account using the +"Sign up" link. +It is not possible to submit a bug report anonymously. -Being now logged in, you can submit a bug. Select the "Create New" link in the -sidebar to open the bug reporting form. +Being now logged in, you can submit an issue. +Click on the "New issue" button in the top bar to report a new issue. -The submission form has a number of fields. For the "Title" field, enter a -*very* short description of the problem; less than ten words is good. In the -"Type" field, select the type of your problem; also select the "Component" and -"Versions" to which the bug relates. +The submission form has two fields, "Title" and "Comment". + +For the "Title" field, enter a *very* short description of the problem; +less than ten words is good. In the "Comment" field, describe the problem in detail, including what you expected to happen and what did happen. Be sure to include whether any extension modules were involved, and what hardware and software platform you were using (including version information as appropriate). -Each bug report will be assigned to a developer who will determine what needs to -be done to correct the problem. You will receive an update each time action is -taken on the bug. +Each issue report will be reviewed by a developer who will determine what needs to +be done to correct the problem. You will receive an update each time an action is +taken on the issue. .. seealso:: @@ -80,7 +81,7 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Writing Guidelines `_ + `Bug Report Writing Guidelines `_ Information about writing a good bug report. Some of this is specific to the Mozilla project, but describes general good practices. @@ -95,6 +96,6 @@ patching Python in the `Python Developer's Guide`_. If you have questions, the `core-mentorship mailing list`_ is a friendly place to get answers to any and all questions pertaining to the process of fixing issues in Python. -.. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity +.. _Documentation bugs: https://github.com/python/cpython/issues?q=is%3Aissue+is%3Aopen+label%3Adocs .. _Python Developer's Guide: https://devguide.python.org/ .. _core-mentorship mailing list: https://mail.python.org/mailman3/lists/core-mentorship.python.org/ diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 8e8a92003c5a3e..33b0c06a9ebc2e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -48,7 +48,7 @@ Allocating Objects on the Heap improving the memory management efficiency. -.. c:function:: void PyObject_Del(PyObject *op) +.. c:function:: void PyObject_Del(void *op) Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index ba9ca5e0d30b92..7a3188009cf959 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -50,7 +50,7 @@ Unless otherwise stated, buffers are not NUL-terminated. Some formats require a read-only :term:`bytes-like object`, and set a pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is *NULL*, +the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is ``NULL``, which disallows mutable objects such as :class:`bytearray`. .. note:: @@ -99,15 +99,15 @@ which disallows mutable objects such as :class:`bytearray`. ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. + pointer is set to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the - ``buf`` member of the :c:type:`Py_buffer` structure is set to *NULL*. + ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. -``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int] +``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. + pointer is set to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a character @@ -124,19 +124,19 @@ which disallows mutable objects such as :class:`bytearray`. bytes-like objects. **This is the recommended way to accept binary data.** -``y#`` (read-only :term:`bytes-like object`) [const char \*, int] +``y#`` (read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`] This variant on ``s#`` doesn't accept Unicode objects, only bytes-like objects. ``S`` (:class:`bytes`) [PyBytesObject \*] Requires that the Python object is a :class:`bytes` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a bytes object. The C variable may also be declared as :c:type:`PyObject\*`. + a bytes object. The C variable may also be declared as :c:type:`PyObject*`. ``Y`` (:class:`bytearray`) [PyByteArrayObject \*] Requires that the Python object is a :class:`bytearray` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`. + a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject*`. ``u`` (:class:`str`) [const Py_UNICODE \*] Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of @@ -151,39 +151,39 @@ which disallows mutable objects such as :class:`bytearray`. Previously, :exc:`TypeError` was raised when embedded null code points were encountered in the Python string. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. -``u#`` (:class:`str`) [const Py_UNICODE \*, int] +``u#`` (:class:`str`) [const Py_UNICODE \*, int or :c:type:`Py_ssize_t`] This variant on ``u`` stores into two C variables, the first one a pointer to a Unicode data buffer, the second one its length. This variant allows null code points. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. ``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*] Like ``u``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. + :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. -``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int] +``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int or :c:type:`Py_ssize_t`] Like ``u#``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. + :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. ``U`` (:class:`str`) [PyObject \*] Requires that the Python object is a Unicode object, without attempting any conversion. Raises :exc:`TypeError` if the object is not a Unicode - object. The C variable may also be declared as :c:type:`PyObject\*`. + object. The C variable may also be declared as :c:type:`PyObject*`. ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer @@ -196,10 +196,10 @@ which disallows mutable objects such as :class:`bytearray`. It only works for encoded data without embedded NUL bytes. This format requires two arguments. The first is only used as input, and - must be a :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + must be a :c:type:`const char*` which points to the name of an encoding as a + NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. @@ -213,16 +213,16 @@ which disallows mutable objects such as :class:`bytearray`. recoding them. Instead, the implementation assumes that the byte string object uses the encoding passed in as parameter. -``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int \*buffer_length] +``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int or :c:type:`Py_ssize_t` \*buffer_length] This variant on ``s#`` is used for encoding Unicode into a character buffer. Unlike the ``es`` format, this variant allows input data which contains NUL characters. It requires three arguments. The first is only used as input, and must be a - :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. + :c:type:`const char*` which points to the name of an encoding as a + NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. The third argument must be a pointer to an integer; the referenced integer @@ -230,12 +230,12 @@ which disallows mutable objects such as :class:`bytearray`. There are two modes of operation: - If *\*buffer* points a *NULL* pointer, the function will allocate a buffer of + If *\*buffer* points a ``NULL`` pointer, the function will allocate a buffer of the needed size, copy the encoded data into this buffer and set *\*buffer* to reference the newly allocated storage. The caller is responsible for calling :c:func:`PyMem_Free` to free the allocated buffer after usage. - If *\*buffer* points to a non-*NULL* pointer (an already allocated buffer), + If *\*buffer* points to a non-``NULL`` pointer (an already allocated buffer), :c:func:`PyArg_ParseTuple` will use this location as the buffer and interpret the initial value of *\*buffer_length* as the buffer size. It will then copy the encoded data into the buffer and NUL-terminate it. If the buffer is not large @@ -244,7 +244,7 @@ which disallows mutable objects such as :class:`bytearray`. In both cases, *\*buffer_length* is set to the length of the encoded data without the trailing NUL byte. -``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int \*buffer_length] +``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int or :c:type:`Py_ssize_t` \*buffer_length] Same as ``es#`` except that byte string objects are passed through without recoding them. Instead, the implementation assumes that the byte string object uses the encoding passed in as parameter. @@ -317,12 +317,12 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not *NULL*. + count is not increased. The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but takes two C arguments: the first is the address of a Python type object, the - second is the address of the C variable (of type :c:type:`PyObject\*`) into which + second is the address of the C variable (of type :c:type:`PyObject*`) into which the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. @@ -331,13 +331,13 @@ Other objects ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C - variable (of arbitrary type), converted to :c:type:`void \*`. The *converter* + variable (of arbitrary type), converted to :c:type:`void *`. The *converter* function in turn is called as follows:: status = converter(object, address); where *object* is the Python object to be converted and *address* is the - :c:type:`void\*` argument that was passed to the :c:func:`PyArg_Parse\*` function. + :c:type:`void*` argument that was passed to the :c:func:`PyArg_Parse\*` function. The returned *status* should be ``1`` for a successful conversion and ``0`` if the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. @@ -345,7 +345,7 @@ Other objects If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second - call, the *object* parameter will be NULL; *address* will have the same value + call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. .. versionchanged:: 3.1 @@ -437,7 +437,7 @@ API Functions Parse the parameters of a function that takes both positional and keyword parameters into local variables. The *keywords* argument is a - *NULL*-terminated array of keyword parameter names. Empty names denote + ``NULL``-terminated array of keyword parameter names. Empty names denote :ref:`positional-only parameters `. Returns true on success; on failure, it returns false and raises the appropriate exception. @@ -483,7 +483,7 @@ API Functions *args*; it must actually be a tuple. The length of the tuple must be at least *min* and no more than *max*; *min* and *max* may be equal. Additional arguments must be passed to the function, each of which should be a pointer to a - :c:type:`PyObject\*` variable; these will be filled in with the values from + :c:type:`PyObject*` variable; these will be filled in with the values from *args*; they will contain borrowed references. The variables which correspond to optional parameters not given by *args* will not be filled in; these should be initialized by the caller. This function returns true on success and false if @@ -520,8 +520,8 @@ Building values Create a new value based on a format string similar to those accepted by the :c:func:`PyArg_Parse\*` family of functions and a sequence of values. Returns - the value or *NULL* in the case of an error; an exception will be raised if - *NULL* is returned. + the value or ``NULL`` in the case of an error; an exception will be raised if + ``NULL`` is returned. :c:func:`Py_BuildValue` does not always build a tuple. It builds a tuple only if its format string contains two or more format units. If the format string is @@ -547,41 +547,41 @@ Building values ``s`` (:class:`str` or ``None``) [const char \*] Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, ``None`` is used. + encoding. If the C string pointer is ``NULL``, ``None`` is used. - ``s#`` (:class:`str` or ``None``) [const char \*, int] + ``s#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`] Convert a C string and its length to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, the length is ignored and + encoding. If the C string pointer is ``NULL``, the length is ignored and ``None`` is returned. ``y`` (:class:`bytes`) [const char \*] This converts a C string to a Python :class:`bytes` object. If the C - string pointer is *NULL*, ``None`` is returned. + string pointer is ``NULL``, ``None`` is returned. - ``y#`` (:class:`bytes`) [const char \*, int] + ``y#`` (:class:`bytes`) [const char \*, int or :c:type:`Py_ssize_t`] This converts a C string and its lengths to a Python object. If the C - string pointer is *NULL*, ``None`` is returned. + string pointer is ``NULL``, ``None`` is returned. ``z`` (:class:`str` or ``None``) [const char \*] Same as ``s``. - ``z#`` (:class:`str` or ``None``) [const char \*, int] + ``z#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`] Same as ``s#``. ``u`` (:class:`str`) [const wchar_t \*] Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) - data to a Python Unicode object. If the Unicode buffer pointer is *NULL*, + data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``, ``None`` is returned. - ``u#`` (:class:`str`) [const wchar_t \*, int] + ``u#`` (:class:`str`) [const wchar_t \*, int or :c:type:`Py_ssize_t`] Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python - Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored + Unicode object. If the Unicode buffer pointer is ``NULL``, the length is ignored and ``None`` is returned. ``U`` (:class:`str` or ``None``) [const char \*] Same as ``s``. - ``U#`` (:class:`str` or ``None``) [const char \*, int] + ``U#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`] Same as ``s#``. ``i`` (:class:`int`) [int] @@ -636,9 +636,9 @@ Building values ``O`` (object) [PyObject \*] Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a *NULL* pointer, it is assumed + incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and - set an exception. Therefore, :c:func:`Py_BuildValue` will return *NULL* but won't + set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is set. @@ -652,8 +652,8 @@ Building values ``O&`` (object) [*converter*, *anything*] Convert *anything* to a Python object through a *converter* function. The - function is called with *anything* (which should be compatible with :c:type:`void - \*`) as its argument and should return a "new" Python object, or *NULL* if an + function is called with *anything* (which should be compatible with :c:type:`void*`) + as its argument and should return a "new" Python object, or ``NULL`` if an error occurred. ``(items)`` (:class:`tuple`) [*matching-items*] @@ -668,7 +668,7 @@ Building values respectively. If there is an error in the format string, the :exc:`SystemError` exception is - set and *NULL* returned. + set and ``NULL`` returned. .. c:function:: PyObject* Py_VaBuildValue(const char *format, va_list vargs) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 72d965053cfda3..dd091bb3d10034 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -89,7 +89,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:type:: Py_buffer - .. c:member:: void \*buf + .. c:member:: void *buf A pointer to the start of the logical structure described by the buffer fields. This can be any location within the underlying physical memory @@ -99,16 +99,16 @@ a buffer, see :c:func:`PyObject_GetBuffer`. For :term:`contiguous` arrays, the value points to the beginning of the memory block. - .. c:member:: void \*obj + .. c:member:: void *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to *NULL* by + the consumer and automatically decremented and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. As a special case, for *temporary* buffers that are wrapped by :c:func:`PyMemoryView_FromBuffer` or :c:func:`PyBuffer_FillInfo` - this field is *NULL*. In general, exporting objects MUST NOT + this field is ``NULL``. In general, exporting objects MUST NOT use this scheme. .. c:member:: Py_ssize_t len @@ -130,25 +130,25 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: Py_ssize_t itemsize Item size in bytes of a single element. Same as the value of :func:`struct.calcsize` - called on non-NULL :c:member:`~Py_buffer.format` values. + called on non-``NULL`` :c:member:`~Py_buffer.format` values. Important exception: If a consumer requests a buffer without the :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_buffer.format` will - be set to *NULL*, but :c:member:`~Py_buffer.itemsize` still has + be set to ``NULL``, but :c:member:`~Py_buffer.itemsize` still has the value for the original format. If :c:member:`~Py_buffer.shape` is present, the equality ``product(shape) * itemsize == len`` still holds and the consumer can use :c:member:`~Py_buffer.itemsize` to navigate the buffer. - If :c:member:`~Py_buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE` + If :c:member:`~Py_buffer.shape` is ``NULL`` as a result of a :c:macro:`PyBUF_SIMPLE` or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char \*format + .. c:member:: const char *format A *NUL* terminated string in :mod:`struct` module style syntax describing - the contents of a single item. If this is *NULL*, ``"B"`` (unsigned bytes) + the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. This field is controlled by the :c:macro:`PyBUF_FORMAT` flag. @@ -158,13 +158,13 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The number of dimensions the memory represents as an n-dimensional array. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` - and :c:member:`~Py_buffer.suboffsets` MUST be *NULL*. + and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions to 64. Exporters MUST respect this limit, consumers of multi-dimensional buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. - .. c:member:: Py_ssize_t \*shape + .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` indicating the shape of the memory as an n-dimensional array. Note that @@ -177,7 +177,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The shape array is read-only for the consumer. - .. c:member:: Py_ssize_t \*strides + .. c:member:: Py_ssize_t *strides An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` giving the number of bytes to skip to get to a new element in each @@ -189,7 +189,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The strides array is read-only for the consumer. - .. c:member:: Py_ssize_t \*suboffsets + .. c:member:: Py_ssize_t *suboffsets An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`. If ``suboffsets[n] >= 0``, the values stored along the nth dimension are @@ -199,7 +199,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. memory block). If all suboffsets are negative (i.e. no de-referencing is needed), then - this field must be NULL (the default value). + this field must be ``NULL`` (the default value). This type of array representation is used by the Python Imaging Library (PIL). See `complex arrays`_ for further information how to access elements @@ -207,7 +207,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The suboffsets array is read-only for the consumer. - .. c:member:: void \*internal + .. c:member:: void *internal This is for use internally by the exporting object. For example, this might be re-cast as an integer by the exporter and used to store flags @@ -248,7 +248,7 @@ readonly, format .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST - be filled in correctly. Otherwise, this field MUST be *NULL*. + be filled in correctly. Otherwise, this field MUST be ``NULL``. :c:macro:`PyBUF_WRITABLE` can be \|'d to any of the flags in the next section. @@ -301,7 +301,7 @@ must be C-contiguous. +-----------------------------------+-------+---------+------------+--------+ | .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F | +-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | C | +| :c:macro:`PyBUF_ND` | yes | NULL | NULL | C | +-----------------------------------+-------+---------+------------+--------+ @@ -349,14 +349,16 @@ The logical structure of NumPy-style arrays is defined by :c:member:`~Py_buffer. If ``ndim == 0``, the memory location pointed to by :c:member:`~Py_buffer.buf` is interpreted as a scalar of size :c:member:`~Py_buffer.itemsize`. In that case, -both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are *NULL*. +both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are ``NULL``. -If :c:member:`~Py_buffer.strides` is *NULL*, the array is interpreted as +If :c:member:`~Py_buffer.strides` is ``NULL``, the array is interpreted as a standard n-dimensional C-array. Otherwise, the consumer must access an n-dimensional array as follows: - ``ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]`` - ``item = *((typeof(item) *)ptr);`` +.. code-block:: c + + ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]; + item = *((typeof(item) *)ptr); As noted above, :c:member:`~Py_buffer.buf` can point to any location within @@ -405,7 +407,7 @@ to two ``char x[2][3]`` arrays that can be located anywhere in memory. Here is a function that returns a pointer to the element in an N-D array -pointed to by an N-dimensional index when there are both non-NULL strides +pointed to by an N-dimensional index when there are both non-``NULL`` strides and suboffsets:: void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, @@ -436,12 +438,12 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set :c:member:`view->obj` to *NULL* and + :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. - On success, fill in *view*, set :c:member:`view->obj` to a new reference + On success, fill in *view*, set ``view->obj`` to a new reference to *exporter* and return 0. In the case of chained buffer providers - that redirect requests to a single object, :c:member:`view->obj` MAY + that redirect requests to a single object, ``view->obj`` MAY refer to this object instead of *exporter* (See :ref:`Buffer Object Structures `). Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls @@ -453,7 +455,7 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) Release the buffer *view* and decrement the reference count for - :c:member:`view->obj`. This function MUST be called when the buffer + ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. It is an error to call this function on a buffer that was not obtained via @@ -473,11 +475,24 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. +.. c:function:: void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices) + + Get the memory area pointed to by the *indices* inside the given *view*. + *indices* must point to an array of ``view->ndim`` indices. + + +.. c:function:: int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort) + + Copy contiguous *len* bytes from *buf* to *view*. + *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) Copy *len* bytes from *src* to its contiguous representation in *buf*. - *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). - ``0`` is returned on success, ``-1`` on error. + *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style + ordering or either one). ``0`` is returned on success, ``-1`` on error. This function fails if *len* != *src->len*. @@ -499,10 +514,10 @@ Buffer-related functions *view* as specified by flags, unless *buf* has been designated as read-only and :c:macro:`PyBUF_WRITABLE` is set in *flags*. - On success, set :c:member:`view->obj` to a new reference to *exporter* and + On success, set ``view->obj`` to a new reference to *exporter* and return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set - :c:member:`view->obj` to *NULL* and return ``-1``; + ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, *exporter* MUST be set to the exporting object and *flags* must be passed - unmodified. Otherwise, *exporter* MUST be NULL. + unmodified. Otherwise, *exporter* MUST be ``NULL``. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index b4a9660a9169e7..b2f409c15abb7a 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -48,7 +48,7 @@ Direct API functions .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) Create a new bytearray object from *string* and its length, *len*. On - failure, *NULL* is returned. + failure, ``NULL`` is returned. .. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) @@ -58,13 +58,13 @@ Direct API functions .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) - Return the size of *bytearray* after checking for a *NULL* pointer. + Return the size of *bytearray* after checking for a ``NULL`` pointer. .. c:function:: char* PyByteArray_AsString(PyObject *bytearray) Return the contents of *bytearray* as a char array after checking for a - *NULL* pointer. The returned array always has an extra + ``NULL`` pointer. The returned array always has an extra null byte appended. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 9a62fb682d6897..0e33ed2c7c9403 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -37,14 +37,14 @@ called with a non-bytes parameter. .. c:function:: PyObject* PyBytes_FromString(const char *v) Return a new bytes object with a copy of the string *v* as value on success, - and *NULL* on failure. The parameter *v* must not be *NULL*; it will not be + and ``NULL`` on failure. The parameter *v* must not be ``NULL``; it will not be checked. .. c:function:: PyObject* PyBytes_FromStringAndSize(const char *v, Py_ssize_t len) Return a new bytes object with a copy of the string *v* as value and length - *len* on success, and *NULL* on failure. If *v* is *NULL*, the contents of + *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. @@ -145,7 +145,7 @@ called with a non-bytes parameter. whether there are any other null bytes. The data must not be modified in any way, unless the object was just created using ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns *NULL* + *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns ``NULL`` and raises :exc:`TypeError`. @@ -159,7 +159,7 @@ called with a non-bytes parameter. Return the null-terminated contents of the object *obj* through the output variables *buffer* and *length*. - If *length* is *NULL*, the bytes object + If *length* is ``NULL``, the bytes object may not contain embedded null bytes; if it does, the function returns ``-1`` and a :exc:`ValueError` is raised. @@ -181,7 +181,7 @@ called with a non-bytes parameter. appended to *bytes*; the caller will own the new reference. The reference to the old value of *bytes* will be stolen. If the new object cannot be created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to *NULL*; the appropriate exception will be set. + of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -201,5 +201,5 @@ called with a non-bytes parameter. desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the reallocation fails, the original bytes object at *\*bytes* is deallocated, - *\*bytes* is set to *NULL*, :exc:`MemoryError` is set, and ``-1`` is + *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 3c921bbde3486f..5eb313c89bfd59 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,7 +15,7 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:type:`void\*` + extension modules who need to pass an opaque value (as a :c:type:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically @@ -40,15 +40,15 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) Create a :c:type:`PyCapsule` encapsulating the *pointer*. The *pointer* - argument may not be *NULL*. + argument may not be ``NULL``. - On failure, set an exception and return *NULL*. + On failure, set an exception and return ``NULL``. - The *name* string may either be *NULL* or a pointer to a valid C string. If - non-*NULL*, this string must outlive the capsule. (Though it is permitted to + The *name* string may either be ``NULL`` or a pointer to a valid C string. If + non-``NULL``, this string must outlive the capsule. (Though it is permitted to free it inside the *destructor*.) - If the *destructor* argument is not *NULL*, it will be called with the + If the *destructor* argument is not ``NULL``, it will be called with the capsule as its argument when it is destroyed. If this capsule will be stored as an attribute of a module, the *name* should @@ -59,20 +59,20 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: void* PyCapsule_GetPointer(PyObject *capsule, const char *name) Retrieve the *pointer* stored in the capsule. On failure, set an exception - and return *NULL*. + and return ``NULL``. The *name* parameter must compare exactly to the name stored in the capsule. - If the name stored in the capsule is *NULL*, the *name* passed in must also - be *NULL*. Python uses the C function :c:func:`strcmp` to compare capsule + If the name stored in the capsule is ``NULL``, the *name* passed in must also + be ``NULL``. Python uses the C function :c:func:`strcmp` to compare capsule names. .. c:function:: PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *capsule) Return the current destructor stored in the capsule. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. - It is legal for a capsule to have a *NULL* destructor. This makes a *NULL* + It is legal for a capsule to have a ``NULL`` destructor. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -80,9 +80,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: void* PyCapsule_GetContext(PyObject *capsule) Return the current context stored in the capsule. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. - It is legal for a capsule to have a *NULL* context. This makes a *NULL* + It is legal for a capsule to have a ``NULL`` context. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -90,9 +90,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: const char* PyCapsule_GetName(PyObject *capsule) Return the current name stored in the capsule. On failure, set an exception - and return *NULL*. + and return ``NULL``. - It is legal for a capsule to have a *NULL* name. This makes a *NULL* return + It is legal for a capsule to have a ``NULL`` name. This makes a ``NULL`` return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or :c:func:`PyErr_Occurred` to disambiguate. @@ -107,13 +107,13 @@ Refer to :ref:`using-capsules` for more information on using these objects. import the module conventionally (using :c:func:`PyImport_ImportModule`). Return the capsule's internal *pointer* on success. On failure, set an - exception and return *NULL*. + exception and return ``NULL``. .. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) Determines whether or not *capsule* is a valid capsule. A valid capsule is - non-*NULL*, passes :c:func:`PyCapsule_CheckExact`, has a non-*NULL* pointer + non-``NULL``, passes :c:func:`PyCapsule_CheckExact`, has a non-``NULL`` pointer stored in it, and its internal name matches the *name* parameter. (See :c:func:`PyCapsule_GetPointer` for information on how capsule names are compared.) @@ -142,9 +142,9 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name) - Set the name inside *capsule* to *name*. If non-*NULL*, the name must + Set the name inside *capsule* to *name*. If non-``NULL``, the name must outlive the capsule. If the previous *name* stored in the capsule was not - *NULL*, no attempt is made to free it. + ``NULL``, no attempt is made to free it. Return ``0`` on success. Return nonzero and set an exception on failure. @@ -152,6 +152,6 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer) Set the void pointer inside *capsule* to *pointer*. The pointer may not be - *NULL*. + ``NULL``. Return ``0`` on success. Return nonzero and set an exception on failure. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 8514afe928e7d3..8408f7e398db7b 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -27,13 +27,13 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: int PyCell_Check(ob) - Return true if *ob* is a cell object; *ob* must not be *NULL*. + Return true if *ob* is a cell object; *ob* must not be ``NULL``. .. c:function:: PyObject* PyCell_New(PyObject *ob) Create and return a new cell object containing the value *ob*. The parameter may - be *NULL*. + be ``NULL``. .. c:function:: PyObject* PyCell_Get(PyObject *cell) @@ -44,19 +44,19 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_GET(PyObject *cell) Return the contents of the cell *cell*, but without checking that *cell* is - non-*NULL* and a cell object. + non-``NULL`` and a cell object. .. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) Set the contents of the cell object *cell* to *value*. This releases the - reference to any current content of the cell. *value* may be *NULL*. *cell* - must be non-*NULL*; if it is not a cell object, ``-1`` will be returned. On + reference to any current content of the cell. *value* may be ``NULL``. *cell* + must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On success, ``0`` will be returned. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) Sets the value of the cell object *cell* to *value*. No reference counts are - adjusted, and no checks are made for safety; *cell* must be non-*NULL* and must + adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must be a cell object. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 48428109e6f2fe..6f8c41ccbf6e85 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -33,19 +33,18 @@ bound into a function. Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyCode_New(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) +.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - Return a new code object. If you need a dummy code object to - create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling - :c:func:`PyCode_New` directly can bind you to a precise Python - version since the definition of the bytecode changes often. + Return a new code object. If you need a dummy code object to create a frame, + use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly + can bind you to a precise Python version since the definition of the bytecode + changes often. - .. versionchanged:: 3.8 - An extra parameter is required (*posonlyargcount*) to support :PEP:`570`. - The first parameter (*argcount*) now represents the total number of positional arguments, - including positional-only. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - .. audit-event:: code.__new__ "code filename name argcount posonlyargcount kwonlyargcount nlocals stacksize flags" + Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments. + + .. versionadded:: 3.8 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index c55f19970e125d..172dcb326a4bc4 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -21,7 +21,7 @@ Codec registry and support functions *object* is passed through the encoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a + be ``NULL`` to use the default method defined for the codec. Raises a :exc:`LookupError` if no encoder can be found. .. c:function:: PyObject* PyCodec_Decode(PyObject *object, const char *encoding, const char *errors) @@ -30,7 +30,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a + be ``NULL`` to use the default method defined for the codec. Raises a :exc:`LookupError` if no encoder can be found. @@ -40,7 +40,7 @@ Codec lookup API In the following functions, the *encoding* string is looked up converted to all lower-case characters, which makes encodings looked up through this mechanism effectively case-insensitive. If no codec is found, a :exc:`KeyError` is set -and *NULL* returned. +and ``NULL`` returned. .. c:function:: PyObject* PyCodec_Encoder(const char *encoding) @@ -92,7 +92,7 @@ Registry API for Unicode encoding error handlers .. c:function:: PyObject* PyCodec_LookupError(const char *name) Lookup the error handling callback function registered under *name*. As a - special case *NULL* can be passed, in which case the error handling callback + special case ``NULL`` can be passed, in which case the error handling callback for "strict" will be returned. .. c:function:: PyObject* PyCodec_StrictErrors(PyObject *exc) diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index f83d711795c2c6..c1d9fa1b41a3fe 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -17,8 +17,8 @@ dictionary, use :c:func:`PyDict_Check`. The chapter is structured like the .. warning:: While the functions described in this chapter carefully check the type of the - objects which are passed in, many of them do not check for *NULL* being passed - instead of a valid object. Allowing *NULL* to be passed in can cause memory + objects which are passed in, many of them do not check for ``NULL`` being passed + instead of a valid object. Allowing ``NULL`` to be passed in can cause memory access violations and immediate termination of the interpreter. diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index a7cde7fb1958a0..314c8b7847e898 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -60,17 +60,17 @@ Type-check macros: .. c:function:: int PyContext_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContext_Type`. *o* must not be - *NULL*. This function always succeeds. + ``NULL``. This function always succeeds. .. c:function:: int PyContextVar_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContextVar_Type`. *o* must not be - *NULL*. This function always succeeds. + ``NULL``. This function always succeeds. .. c:function:: int PyContextToken_CheckExact(PyObject *o) Return true if *o* is of type :c:data:`PyContextToken_Type`. - *o* must not be *NULL*. This function always succeeds. + *o* must not be ``NULL``. This function always succeeds. Context object management functions: @@ -112,9 +112,9 @@ Context variable functions: .. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def) Create a new ``ContextVar`` object. The *name* parameter is used - for introspection and debug purposes. The *def* parameter may optionally - specify the default value for the context variable. If an error has - occurred, this function returns ``NULL``. + for introspection and debug purposes. The *def* parameter specifies + a default value for the context variable, or ``NULL`` for no default. + If an error has occurred, this function returns ``NULL``. .. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) @@ -129,13 +129,12 @@ Context variable functions: - the default value of *var*, if not ``NULL``; - ``NULL`` - If the value was found, the function will create a new reference to it. + Except for ``NULL``, the function returns a new reference. .. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value) - Set the value of *var* to *value* in the current context. Returns a - pointer to a :c:type:`PyObject` object, or ``NULL`` if an error - has occurred. + Set the value of *var* to *value* in the current context. Returns + a new token object for this change, or ``NULL`` if an error has occurred. .. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index ed101c791ec743..0cc836b70a018f 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -11,40 +11,40 @@ Functions for number conversion and formatted string output. .. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) Output not more than *size* bytes to *str* according to the format string - *format* and the extra arguments. See the Unix man page :manpage:`snprintf(2)`. + *format* and the extra arguments. See the Unix man page :manpage:`snprintf(3)`. .. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) Output not more than *size* bytes to *str* according to the format string *format* and the variable argument list *va*. Unix man page - :manpage:`vsnprintf(2)`. + :manpage:`vsnprintf(3)`. :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to guarantee consistent behavior in corner cases, which the Standard C functions do not. -The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They +The wrappers ensure that ``str[size-1]`` is always ``'\0'`` upon return. They never write more than *size* bytes (including the trailing ``'\0'``) into str. Both functions require that ``str != NULL``, ``size > 0`` and ``format != NULL``. If the platform doesn't have :c:func:`vsnprintf` and the buffer size needed to avoid truncation exceeds *size* by more than 512 bytes, Python aborts with a -*Py_FatalError*. +:c:func:`Py_FatalError`. The return value (*rv*) for these functions should be interpreted as follows: * When ``0 <= rv < size``, the output conversion was successful and *rv* characters were written to *str* (excluding the trailing ``'\0'`` byte at - *str*[*rv*]). + ``str[rv]``). * When ``rv >= size``, the output conversion was truncated and a buffer with - ``rv + 1`` bytes would have been needed to succeed. *str*[*size*-1] is ``'\0'`` + ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." *str*[*size*-1] is ``'\0'`` in +* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -95,25 +95,25 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values *Py_DTSF_SIGN*, - *Py_DTSF_ADD_DOT_0*, or *Py_DTSF_ALT*, or-ed together: + *flags* can be zero or more of the values ``Py_DTSF_SIGN``, + ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: - * *Py_DTSF_SIGN* means to always precede the returned string with a sign + * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign character, even if *val* is non-negative. - * *Py_DTSF_ADD_DOT_0* means to ensure that the returned string will not look + * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look like an integer. - * *Py_DTSF_ALT* means to apply "alternate" formatting rules. See the + * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for details. - If *ptype* is non-NULL, then the value it points to will be set to one of - *Py_DTST_FINITE*, *Py_DTST_INFINITE*, or *Py_DTST_NAN*, signifying that + If *ptype* is non-``NULL``, then the value it points to will be set to one of + ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that *val* is a finite number, an infinite number, or not a number, respectively. The return value is a pointer to *buffer* with the converted string or - *NULL* if the conversion failed. The caller is responsible for freeing the + ``NULL`` if the conversion failed. The caller is responsible for freeing the returned string by calling :c:func:`PyMem_Free`. .. versionadded:: 3.1 diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index 50428c7eb43e3d..2260944a9a93a9 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -23,7 +23,7 @@ return. .. c:function:: int PyCoro_CheckExact(PyObject *ob) - Return true if *ob*'s type is *PyCoro_Type*; *ob* must not be *NULL*. + Return true if *ob*'s type is :c:type:`PyCoro_Type`; *ob* must not be ``NULL``. .. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) @@ -31,4 +31,4 @@ return. Create and return a new coroutine object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. + must not be ``NULL``. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 44a04373d19683..84e05ea631eadb 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -28,61 +28,61 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. .. c:function:: int PyDate_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. .. c:function:: int PyDateTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be *NULL*. + be ``NULL``. .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. .. c:function:: int PyTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. .. c:function:: int PyDelta_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - *NULL*. + ``NULL``. .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be *NULL*. + :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. .. c:function:: int PyTZInfo_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - *NULL*. + ``NULL``. Macros to create objects: @@ -144,7 +144,7 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of :c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be *NULL*, and the type is +:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -164,7 +164,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument -must not be *NULL*, and the type is not checked: +must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -186,8 +186,15 @@ must not be *NULL*, and the type is not checked: Return the microsecond, as an int from 0 through 999999. +.. c:function:: int PyDateTime_DATE_GET_FOLD(PyDateTime_DateTime *o) + + Return the fold, as an int from 0 through 1. + + .. versionadded:: 3.6 + + Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be *NULL*, +:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -210,9 +217,16 @@ and the type is not checked: Return the microsecond, as an int from 0 through 999999. +.. c:function:: int PyDateTime_TIME_GET_FOLD(PyDateTime_Time *o) + + Return the fold, as an int from 0 through 1. + + .. versionadded:: 3.6 + + Macros to extract fields from time delta objects. The argument must be an instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must -not be *NULL*, and the type is not checked: +not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e970771893eb30..e910324ef8b5ac 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -33,7 +33,7 @@ Dictionary Objects .. c:function:: PyObject* PyDict_New() - Return a new empty dictionary, or *NULL* on failure. + Return a new empty dictionary, or ``NULL`` on failure. .. c:function:: PyObject* PyDictProxy_New(PyObject *mapping) @@ -62,37 +62,40 @@ Dictionary Objects .. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) - Insert *value* into the dictionary *p* with a key of *key*. *key* must be + Insert *val* into the dictionary *p* with a key of *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. + ``0`` on success or ``-1`` on failure. This function *does not* steal a + reference to *val*. .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) .. index:: single: PyUnicode_FromString() - Insert *value* into the dictionary *p* using *key* as a key. *key* should - be a :c:type:`const char\*`. The key object is created using + Insert *val* into the dictionary *p* using *key* as a key. *key* should + be a :c:type:`const char*`. The key object is created using ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. + failure. This function *does not* steal a reference to *val*. .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) Remove the entry in dictionary *p* with key *key*. *key* must be hashable; - if it isn't, :exc:`TypeError` is raised. Return ``0`` on success or ``-1`` - on failure. + if it isn't, :exc:`TypeError` is raised. + If *key* is not in the dictionary, :exc:`KeyError` is raised. + Return ``0`` on success or ``-1`` on failure. .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the string - *key*. Return ``0`` on success or ``-1`` on failure. + Remove the entry in dictionary *p* which has a key specified by the string *key*. + If *key* is not in the dictionary, :exc:`KeyError` is raised. + Return ``0`` on success or ``-1`` on failure. .. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) - Return the object from dictionary *p* which has a key *key*. Return *NULL* + Return the object from dictionary *p* which has a key *key*. Return ``NULL`` if the key *key* is not present, but *without* setting an exception. Note that exceptions which occur while calling :meth:`__hash__` and @@ -103,15 +106,15 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) Variant of :c:func:`PyDict_GetItem` that does not suppress - exceptions. Return *NULL* **with** an exception set if an exception - occurred. Return *NULL* **without** an exception set if the key + exceptions. Return ``NULL`` **with** an exception set if an exception + occurred. Return ``NULL`` **without** an exception set if the key wasn't present. .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:type:`const char\*`, rather than a :c:type:`PyObject\*`. + :c:type:`const char*`, rather than a :c:type:`PyObject*`. Note that exceptions which occur while calling :meth:`__hash__` and :meth:`__eq__` methods and creating a temporary string object @@ -160,8 +163,8 @@ Dictionary Objects prior to the first call to this function to start the iteration; the function returns true for each pair in the dictionary, and false once all pairs have been reported. The parameters *pkey* and *pvalue* should either - point to :c:type:`PyObject\*` variables that will be filled in with each key - and value, respectively, or may be *NULL*. Any references returned through + point to :c:type:`PyObject*` variables that will be filled in with each key + and value, respectively, or may be ``NULL``. Any references returned through them are borrowed. *ppos* should not be altered during iteration. Its value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d3f6daa8347e41..a4c263f47ab63d 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -13,15 +13,15 @@ exception handling. It works somewhat like the POSIX :c:data:`errno` variable: there is a global indicator (per thread) of the last error that occurred. Most C API functions don't clear this on success, but will set it to indicate the cause of the error on failure. Most C API functions also return an error -indicator, usually *NULL* if they are supposed to return a pointer, or ``-1`` +indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` if they return an integer (exception: the :c:func:`PyArg_\*` functions return ``1`` for success and ``0`` for failure). Concretely, the error indicator consists of three object pointers: the exception's type, the exception's value, and the traceback object. Any -of those pointers can be NULL if non-set (although some combinations are -forbidden, for example you can't have a non-NULL traceback if the exception -type is NULL). +of those pointers can be ``NULL`` if non-set (although some combinations are +forbidden, for example you can't have a non-``NULL`` traceback if the exception +type is ``NULL``). When a function must fail because some function it called failed, it generally doesn't set the error indicator; the function it called already set it. It is @@ -92,7 +92,7 @@ Raising exceptions These functions help you set the current thread's error indicator. For convenience, some of these functions will always return a -NULL pointer for use in a ``return`` statement. +``NULL`` pointer for use in a ``return`` statement. .. c:function:: void PyErr_SetString(PyObject *type, const char *message) @@ -111,7 +111,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - This function sets the error indicator and returns *NULL*. *exception* + This function sets the error indicator and returns ``NULL``. *exception* should be a Python exception class. The *format* and subsequent parameters help format the error message; they have the same meaning and values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded @@ -140,7 +140,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_NoMemory() - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns *NULL* + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` so an object allocation function can write ``return PyErr_NoMemory();`` when it runs out of memory. @@ -156,7 +156,7 @@ NULL pointer for use in a ``return`` statement. and then calls ``PyErr_SetObject(type, object)``. On Unix, when the :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns *NULL*, so a wrapper + leaves it set to that. The function always returns ``NULL``, so a wrapper function around a system call can write ``return PyErr_SetFromErrno(type);`` when the system call returns an error. @@ -164,7 +164,7 @@ NULL pointer for use in a ``return`` statement. .. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not *NULL*, it is passed to the constructor of *type* as + *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as a third parameter. In the case of :exc:`OSError` exception, this is used to define the :attr:`filename` attribute of the exception instance. @@ -195,7 +195,7 @@ NULL pointer for use in a ``return`` statement. then it constructs a tuple object whose first item is the *ierr* value and whose second item is the corresponding error message (gotten from :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns *NULL*. + object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -304,7 +304,7 @@ an error value). .. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) Issue a warning message. The *category* argument is a warning category (see - below) or *NULL*; the *message* argument is a UTF-8 encoded string. *stack_level* is a + below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a positive number giving a number of stack frames; the warning will be issued from the currently executing line of code in that stack frame. A *stack_level* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, @@ -320,7 +320,7 @@ an error value). :mod:`warnings` module and the :option:`-W` option in the command line documentation. There is no C API for warning control. -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path) +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) Much like :c:func:`PyErr_SetImportError` but this function allows for specifying a subclass of :exc:`ImportError` to raise. @@ -333,7 +333,7 @@ an error value). Issue a warning message with explicit control over all warning attributes. This is a straightforward wrapper around the Python function :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to *NULL* to get the default effect + and *registry* arguments may be set to ``NULL`` to get the default effect described there. .. versionadded:: 3.4 @@ -358,7 +358,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. .. versionadded:: 3.6 @@ -370,7 +370,7 @@ Querying the error indicator Test whether the error indicator is set. If set, return the exception *type* (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not + functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not own a reference to the return value, so you do not need to :c:func:`Py_DECREF` it. @@ -400,9 +400,9 @@ Querying the error indicator .. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to *NULL*. If it is + If the error indicator is not set, set all three variables to ``NULL``. If it is set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be *NULL* even when the type object is not. + value and traceback object may be ``NULL`` even when the type object is not. .. note:: @@ -422,8 +422,8 @@ Querying the error indicator .. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are *NULL*, the error - indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or + already set, it is cleared first. If the objects are ``NULL``, the error + indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or traceback. The exception type should be a class. Do not pass an invalid exception type or value. (Violating these rules will cause subtle problems later.) This call takes away a reference to each object: you must own a @@ -462,7 +462,7 @@ Querying the error indicator Retrieve the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was freshly raised. Returns new references for the three objects, any of which - may be *NULL*. Does not modify the exception info state. + may be ``NULL``. Does not modify the exception info state. .. note:: @@ -479,7 +479,7 @@ Querying the error indicator Set the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was freshly raised. This function steals the references of the arguments. - To clear the exception state, pass *NULL* for all three arguments. + To clear the exception state, pass ``NULL`` for all three arguments. For general rules about the three arguments, see :c:func:`PyErr_Restore`. .. note:: @@ -549,7 +549,7 @@ Exception Classes This utility function creates and returns a new exception class. The *name* argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally *NULL*. + ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). @@ -563,7 +563,7 @@ Exception Classes .. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-*NULL*, it will be used as the + easily be given a docstring: If *doc* is non-``NULL``, it will be used as the docstring for the exception class. .. versionadded:: 3.2 @@ -576,7 +576,7 @@ Exception Objects Return the traceback associated with the exception as a new reference, as accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns *NULL*. + traceback associated, this returns ``NULL``. .. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) @@ -590,12 +590,12 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from Python through :attr:`__context__`. If there is no context associated, this - returns *NULL*. + returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - Set the context associated with the exception to *ctx*. Use *NULL* to clear + Set the context associated with the exception to *ctx*. Use ``NULL`` to clear it. There is no type check to make sure that *ctx* is an exception instance. This steals a reference to *ctx*. @@ -609,7 +609,7 @@ Exception Objects .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - Set the cause associated with the exception to *cause*. Use *NULL* to clear + Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception instance or :const:`None`. This steals a reference to *cause*. @@ -635,11 +635,21 @@ The following functions are used to create and modify Unicode exceptions from C. *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are UTF-8 encoded strings. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + .. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) Create a :class:`UnicodeTranslateError` object with the attributes *object*, *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + .. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) @@ -656,7 +666,7 @@ The following functions are used to create and modify Unicode exceptions from C. int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be *NULL*. Return ``0`` on success, ``-1`` on + *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) @@ -671,7 +681,7 @@ The following functions are used to create and modify Unicode exceptions from C. int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be *NULL*. Return ``0`` on success, ``-1`` on + *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -761,7 +771,7 @@ Standard Exceptions All standard Python exceptions are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: @@ -809,7 +819,6 @@ the variables: single: PyExc_SystemError single: PyExc_SystemExit single: PyExc_TabError - single: PyExc_TargetScopeError single: PyExc_TimeoutError single: PyExc_TypeError single: PyExc_UnboundLocalError @@ -911,8 +920,6 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TabError` | :exc:`TabError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TargetScopeError` | :exc:`TargetScopeError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_TypeError` | :exc:`TypeError` | | @@ -974,9 +981,6 @@ Notes: This is a base class for other standard exceptions. (2) - This is the same as :exc:`weakref.ReferenceError`. - -(3) Only defined on Windows; protect code that uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. @@ -987,7 +991,7 @@ Standard Warning Categories All standard Python warning categories are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 543dc60b9f55ab..ed3735aa83608a 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -8,7 +8,7 @@ File Objects .. index:: object: file These APIs are a minimal emulation of the Python 2 C API for built-in file -objects, which used to rely on the buffered I/O (:c:type:`FILE\*`) support +objects, which used to rely on the buffered I/O (:c:type:`FILE*`) support from the C standard library. In Python 3, files and streams use the new :mod:`io` module, which defines several layers over the low-level unbuffered I/O of the operating system. The functions described below are @@ -17,13 +17,13 @@ error reporting in the interpreter; third-party code is advised to access the :mod:`io` APIs instead. -.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) +.. c:function:: PyObject* PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) Create a Python file object from the file descriptor of an already opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* - can be *NULL* to use the defaults; *buffering* can be *-1* to use the + can be ``NULL`` to use the defaults; *buffering* can be *-1* to use the default. *name* is ignored and kept for backward compatibility. Return - *NULL* on failure. For a more comprehensive description of the arguments, + ``NULL`` on failure. For a more comprehensive description of the arguments, please refer to the :func:`io.open` function documentation. .. warning:: @@ -82,6 +82,8 @@ the :mod:`io` APIs instead. This function is safe to call before :c:func:`Py_Initialize`. + .. audit-event:: setopencodehook "" c.PyFile_SetOpenCodeHook + .. versionadded:: 3.8 diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 057ff522516a3a..bfc28a79ecfdc7 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -34,12 +34,12 @@ Floating Point Objects .. c:function:: PyObject* PyFloat_FromString(PyObject *str) Create a :c:type:`PyFloatObject` object based on the string value in *str*, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyFloat_FromDouble(double v) - Create a :c:type:`PyFloatObject` object from *v*, or *NULL* on failure. + Create a :c:type:`PyFloatObject` object from *v*, or ``NULL`` on failure. .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 02c4ebdbed4546..bb416f4bb63aa2 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -26,7 +26,7 @@ There are a few functions specific to Python functions. .. c:function:: int PyFunction_Check(PyObject *o) Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be *NULL*. + The parameter must not be ``NULL``. .. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) @@ -36,14 +36,14 @@ There are a few functions specific to Python functions. The function's docstring and name are retrieved from the code object. *__module__* is retrieved from *globals*. The argument defaults, annotations and closure are - set to *NULL*. *__qualname__* is set to the same value as the function's name. + set to ``NULL``. *__qualname__* is set to the same value as the function's name. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or NULL; - if NULL, the ``__qualname__`` attribute is set to the same value as its + ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the ``__qualname__`` attribute is set to the same value as its ``__name__`` attribute. .. versionadded:: 3.3 @@ -69,27 +69,27 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) Return the argument default values of the function object *op*. This can be a - tuple of arguments or *NULL*. + tuple of arguments or ``NULL``. .. c:function:: int PyFunction_SetDefaults(PyObject *op, PyObject *defaults) Set the argument default values for the function object *op*. *defaults* must be - *Py_None* or a tuple. + ``Py_None`` or a tuple. Raises :exc:`SystemError` and returns ``-1`` on failure. .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) - Return the closure associated with the function object *op*. This can be *NULL* + Return the closure associated with the function object *op*. This can be ``NULL`` or a tuple of cell objects. .. c:function:: int PyFunction_SetClosure(PyObject *op, PyObject *closure) Set the closure associated with the function object *op*. *closure* must be - *Py_None* or a tuple of cell objects. + ``Py_None`` or a tuple of cell objects. Raises :exc:`SystemError` and returns ``-1`` on failure. @@ -97,12 +97,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject *PyFunction_GetAnnotations(PyObject *op) Return the annotations of the function object *op*. This can be a - mutable dictionary or *NULL*. + mutable dictionary or ``NULL``. .. c:function:: int PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) Set the annotations for the function object *op*. *annotations* - must be a dictionary or *Py_None*. + must be a dictionary or ``Py_None``. Raises :exc:`SystemError` and returns ``-1`` on failure. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index b3f30b2ed9e999..924a7fd2fda4d5 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -49,7 +49,7 @@ Constructors for container types must conform to two rules: .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the - resized object or *NULL* on failure. *op* must not be tracked by the collector yet. + resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. .. c:function:: void PyObject_GC_Track(PyObject *op) @@ -110,7 +110,7 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: Traversal function for a container object. Implementations must call the *visit* function for each object directly contained by *self*, with the parameters to *visit* being the contained object and the *arg* value passed - to the handler. The *visit* function must not be called with a *NULL* + to the handler. The *visit* function must not be called with a ``NULL`` object argument. If *visit* returns a non-zero value that value should be returned immediately. @@ -121,7 +121,7 @@ must name its arguments exactly *visit* and *arg*: .. c:function:: void Py_VISIT(PyObject *o) - If *o* is not *NULL*, call the *visit* callback, with arguments *o* + If *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: @@ -134,7 +134,7 @@ must name its arguments exactly *visit* and *arg*: return 0; } -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` if the object is immutable. diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 8d54021c181457..74410927bfde10 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -22,23 +22,23 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: int PyGen_Check(PyObject *ob) - Return true if *ob* is a generator object; *ob* must not be *NULL*. + Return true if *ob* is a generator object; *ob* must not be ``NULL``. .. c:function:: int PyGen_CheckExact(PyObject *ob) - Return true if *ob*'s type is *PyGen_Type*; *ob* must not be *NULL*. + Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be ``NULL``. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) Create and return a new generator object based on the *frame* object. A reference to *frame* is stolen by this function. The argument must not be - *NULL*. + ``NULL``. .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. + must not be ``NULL``. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 86cc4031610b7d..78f935a2911bc6 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -14,7 +14,7 @@ Importing Modules single: modules (in module sys) This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to *NULL* and *level* set + leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set to 0. When the *name* argument contains a dot (when it specifies a submodule of a package), the *fromlist* argument is set to the list ``['*']`` so that the return value is the @@ -22,7 +22,7 @@ Importing Modules be the case. (Unfortunately, this has an additional side effect when *name* in fact specifies a subpackage instead of a submodule: the submodules specified in the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or *NULL* with an exception set on failure. A failing + imported module, or ``NULL`` with an exception set on failure. A failing import of a module doesn't leave the module in :data:`sys.modules`. This function always uses absolute imports. @@ -47,7 +47,7 @@ Importing Modules function :func:`__import__`. The return value is a new reference to the imported module or top-level - package, or *NULL* with an exception set on failure. Like for + package, or ``NULL`` with an exception set on failure. Like for :func:`__import__`, the return value when a submodule of a package was requested is normally the top-level package, unless a non-empty *fromlist* was given. @@ -63,7 +63,7 @@ Importing Modules this function directly. The return value is a new reference to the imported module or top-level package, - or *NULL* with an exception set on failure. Like for :func:`__import__`, + or ``NULL`` with an exception set on failure. Like for :func:`__import__`, the return value when a submodule of a package was requested is normally the top-level package, unless a non-empty *fromlist* was given. @@ -91,7 +91,7 @@ Importing Modules .. c:function:: PyObject* PyImport_ReloadModule(PyObject *m) - Reload a module. Return a new reference to the reloaded module, or *NULL* with + Reload a module. Return a new reference to the reloaded module, or ``NULL`` with an exception set on failure (the module still exists in this case). @@ -100,7 +100,7 @@ Importing Modules Return the module object corresponding to a module name. The *name* argument may be of the form ``package.module``. First check the modules dictionary if there's one there, and if not, create a new one and insert it in the modules - dictionary. Return *NULL* with an exception set on failure. + dictionary. Return ``NULL`` with an exception set on failure. .. note:: @@ -125,7 +125,7 @@ Importing Modules Given a module name (possibly of the form ``package.module``) and a code object read from a Python bytecode file or obtained from the built-in function :func:`compile`, load the module. Return a new reference to the module object, - or *NULL* with an exception set if an error occurred. *name* + or ``NULL`` with an exception set if an error occurred. *name* is removed from :attr:`sys.modules` in error cases, even if *name* was already in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of @@ -207,8 +207,8 @@ Importing Modules .. c:function:: PyObject* PyImport_GetModule(PyObject *name) Return the already imported module with the given name. If the - module has not been imported yet then returns NULL but does not set - an error. Returns NULL and sets an error if the lookup failed. + module has not been imported yet then returns ``NULL`` but does not set + an error. Returns ``NULL`` and sets an error if the lookup failed. .. versionadded:: 3.7 @@ -277,7 +277,7 @@ Importing Modules .. c:var:: const struct _frozen* PyImport_FrozenModules This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen + records, terminated by one whose members are all ``NULL`` or zero. When a frozen module is imported, it is searched in this table. Third-party code could play tricks with this to provide a dynamically created collection of frozen modules. @@ -310,7 +310,7 @@ Importing Modules .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) Add a collection of modules to the table of built-in modules. The *newtab* - array must end with a sentinel entry which contains *NULL* for the :attr:`name` + array must end with a sentinel entry which contains ``NULL`` for the :attr:`name` field; failure to provide the sentinel value can result in a memory fault. Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to extend the internal table. In the event of failure, no modules are added to the diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index a0ac4d21d13936..e12f7c6262c22f 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,8 @@ Initialization, Finalization, and Threads ***************************************** +See also :ref:`Python Initialization Configuration `. + .. _pre-init-safe: Before Python Initialization @@ -79,7 +81,7 @@ When a flag is set by an option, the value of the flag is the number of times that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. -.. c:var:: Py_BytesWarningFlag +.. c:var:: int Py_BytesWarningFlag Issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater @@ -87,7 +89,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. -.. c:var:: Py_DebugFlag +.. c:var:: int Py_DebugFlag Turn on parser debugging output (for expert only, depending on compilation options). @@ -95,7 +97,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. -.. c:var:: Py_DontWriteBytecodeFlag +.. c:var:: int Py_DontWriteBytecodeFlag If set to non-zero, Python won't try to write ``.pyc`` files on the import of source modules. @@ -103,14 +105,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. -.. c:var:: Py_FrozenFlag +.. c:var:: int Py_FrozenFlag Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. -.. c:var:: Py_HashRandomizationFlag +.. c:var:: int Py_HashRandomizationFlag Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to a non-empty string. @@ -118,14 +120,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. -.. c:var:: Py_IgnoreEnvironmentFlag +.. c:var:: int Py_IgnoreEnvironmentFlag Ignore all :envvar:`PYTHON*` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. Set by the :option:`-E` and :option:`-I` options. -.. c:var:: Py_InspectFlag +.. c:var:: int Py_InspectFlag When a script is passed as first argument or the :option:`-c` option is used, enter interactive mode after executing the script or the command, even when @@ -134,11 +136,11 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. -.. c:var:: Py_InteractiveFlag +.. c:var:: int Py_InteractiveFlag Set by the :option:`-i` option. -.. c:var:: Py_IsolatedFlag +.. c:var:: int Py_IsolatedFlag Run Python in isolated mode. In isolated mode :data:`sys.path` contains neither the script's directory nor the user's site-packages directory. @@ -147,7 +149,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 -.. c:var:: Py_LegacyWindowsFSEncodingFlag +.. c:var:: int Py_LegacyWindowsFSEncodingFlag If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 encoding for the filesystem encoding. @@ -159,7 +161,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_LegacyWindowsStdioFlag +.. c:var:: int Py_LegacyWindowsStdioFlag If the flag is non-zero, use :class:`io.FileIO` instead of :class:`WindowsConsoleIO` for :mod:`sys` standard streams. @@ -171,7 +173,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_NoSiteFlag +.. c:var:: int Py_NoSiteFlag Disable the import of the module :mod:`site` and the site-dependent manipulations of :data:`sys.path` that it entails. Also disable these @@ -180,7 +182,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. -.. c:var:: Py_NoUserSiteDirectory +.. c:var:: int Py_NoUserSiteDirectory Don't add the :data:`user site-packages directory ` to :data:`sys.path`. @@ -188,12 +190,12 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. -.. c:var:: Py_OptimizeFlag +.. c:var:: int Py_OptimizeFlag Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. -.. c:var:: Py_QuietFlag +.. c:var:: int Py_QuietFlag Don't display the copyright and version messages even in interactive mode. @@ -201,14 +203,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 -.. c:var:: Py_UnbufferedStdioFlag +.. c:var:: int Py_UnbufferedStdioFlag Force the stdout and stderr streams to be unbuffered. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. -.. c:var:: Py_VerboseFlag +.. c:var:: int Py_VerboseFlag Print a message each time a module is initialized, showing the place (filename or built-in module) from which it is loaded. If greater or equal @@ -299,8 +301,9 @@ Initializing and finalizing the interpreter than once; this can happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` more than once. - .. versionadded:: 3.6 + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + .. versionadded:: 3.6 .. c:function:: void Py_Finalize() @@ -326,7 +329,7 @@ Process-wide parameters It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code to control IO encoding when the environment variable does not work. - ``encoding`` and/or ``errors`` may be NULL to use + *encoding* and/or *errors* may be ``NULL`` to use :envvar:`PYTHONIOENCODING` and/or default values (depending on other settings). @@ -469,8 +472,8 @@ Process-wide parameters dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` on Windows. - This also causes :data:`sys.executable` to be set only to the raw program - name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and + This also causes :data:`sys.executable` to be set to the program + full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these if required after calling :c:func:`Py_Initialize`. @@ -480,6 +483,10 @@ Process-wide parameters The path argument is copied internally, so the caller may free it after the call completes. + .. versionchanged:: 3.8 + The program full path is now used for :data:`sys.executable`, instead + of the program name. + .. c:function:: const char* Py_GetVersion() @@ -762,9 +769,19 @@ supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the :c:func:`PyGILState_\*` API is unsupported. + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + Another important thing to note about threads is their behaviour in the face of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. That also +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains means any locks held by other threads will never be released. Python solves this for :func:`os.fork` by acquiring the locks it uses internally before the fork, and releasing them afterwards. In addition, it resets any @@ -779,6 +796,17 @@ being held by a thread that is defunct after the fork. :c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not always able to. +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + High-level API -------------- @@ -802,7 +830,7 @@ code, or when embedding the Python interpreter: .. c:type:: PyThreadState This data structure represents the state of a single thread. The only public - data member is :c:type:`PyInterpreterState \*`:attr:`interp`, which points to + data member is :attr:`interp` (:c:type:`PyInterpreterState *`), which points to this thread's interpreter state. @@ -844,15 +872,15 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyEval_SaveThread() Release the global interpreter lock (if it has been created) and reset the - thread state to *NULL*, returning the previous thread state (which is not - *NULL*). If the lock has been created, the current thread must have + thread state to ``NULL``, returning the previous thread state (which is not + ``NULL``). If the lock has been created, the current thread must have acquired it. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) Acquire the global interpreter lock (if it has been created) and set the - thread state to *tstate*, which must not be *NULL*. If the lock has been + thread state to *tstate*, which must not be ``NULL``. If the lock has been created, the current thread must not have acquired it, otherwise deadlock ensues. @@ -866,14 +894,14 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyThreadState_Get() Return the current thread state. The global interpreter lock must be held. - When the current thread state is *NULL*, this issues a fatal error (so that - the caller needn't check for *NULL*). + When the current thread state is ``NULL``, this issues a fatal error (so that + the caller needn't check for ``NULL``). .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) Swap the current thread state with the thread state given by the argument - *tstate*, which may be *NULL*. The global interpreter lock must be held + *tstate*, which may be ``NULL``. The global interpreter lock must be held and is not released. @@ -990,12 +1018,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`. be held, but may be held if it is necessary to serialize calls to this function. + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) Reset all information in an interpreter state object. The global interpreter lock must be held. + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) @@ -1035,7 +1067,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) Return a dictionary in which interpreter-specific data may be stored. - If this function returns *NULL* then no exception has been raised and + If this function returns ``NULL`` then no exception has been raised and the caller should assume no interpreter-specific dict is available. This is not a replacement for :c:func:`PyModule_GetState()`, which @@ -1049,7 +1081,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in the dictionary. It is okay to call this function when no current thread state - is available. If this function returns *NULL*, no exception has been raised and + is available. If this function returns ``NULL``, no exception has been raised and the caller should assume no current thread state is available. @@ -1070,7 +1102,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) Acquire the global interpreter lock and set the current thread state to - *tstate*, which should not be *NULL*. The lock must have been created earlier. + *tstate*, which should not be ``NULL``. The lock must have been created earlier. If this thread already has the lock, deadlock ensues. .. note:: @@ -1091,9 +1123,9 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - Reset the current thread state to *NULL* and release the global interpreter + Reset the current thread state to ``NULL`` and release the global interpreter lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be *NULL*, is only used to check + thread. The *tstate* argument, which must not be ``NULL``, is only used to check that it represents the current thread state --- if it isn't, a fatal error is reported. @@ -1141,10 +1173,18 @@ Sub-interpreter support While in most uses, you will only embed a single Python interpreter, there are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. You can switch between sub-interpreters using the -:c:func:`PyThreadState_Swap` function. You can create and destroy them -using the following functions: +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: .. c:function:: PyThreadState* Py_NewInterpreter() @@ -1170,7 +1210,7 @@ using the following functions: The return value points to the first thread state created in the new sub-interpreter. This thread state is made in the current thread state. Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, *NULL* is + below. If creation of the new interpreter is unsuccessful, ``NULL`` is returned; no exception is set since the exception state is stored in the current thread state and there may not be a current thread state. (Like all other Python/C API functions, the global interpreter lock must be held before @@ -1182,15 +1222,31 @@ using the following functions: single: Py_FinalizeEx() single: Py_Initialize() - Extension modules are shared between (sub-)interpreters as follows: the first - time a particular extension is imported, it is initialized normally, and a - (shallow) copy of its module's dictionary is squirreled away. When the same - extension is imported by another (sub-)interpreter, a new module is initialized - and filled with the contents of this copy; the extension's ``init`` function is - not called. Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's - ``initmodule`` function *is* called again. + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using single-phase initialization, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. .. index:: single: close() (in module os) @@ -1201,7 +1257,7 @@ using the following functions: Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread - states below. When the call returns, the current thread state is *NULL*. All + states below. When the call returns, the current thread state is ``NULL``. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that @@ -1216,14 +1272,16 @@ process, the insulation between them isn't perfect --- for example, using low-level file operations like :func:`os.close` they can (accidentally or maliciously) affect each other's open files. Because of the way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when the extension makes use of -(static) global variables, or when the extension manipulates its module's -dictionary after its initialization. It is possible to insert objects created -in one sub-interpreter into a namespace of another sub-interpreter; this should -be done with great care to avoid sharing user-defined functions, methods, -instances or classes between sub-interpreters, since import operations executed -by such objects may affect the wrong (sub-)interpreter's dictionary of loaded -modules. +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. Also note that combining this functionality with :c:func:`PyGILState_\*` APIs is delicate, because these APIs assume a bijection between Python thread states @@ -1308,27 +1366,27 @@ Python-level trace functions in previous versions. :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`, or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - +------------------------------+--------------------------------------+ - | Value of *what* | Meaning of *arg* | - +==============================+======================================+ - | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_RETURN` | Value being returned to the caller, | - | | or *NULL* if caused by an exception. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_CALL` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_EXCEPTION` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_RETURN` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ + +------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +==============================+========================================+ + | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_CALL` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_EXCEPTION` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_C_RETURN` | Function object being called. | + +------------------------------+----------------------------------------+ + | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +------------------------------+----------------------------------------+ .. c:var:: int PyTrace_CALL @@ -1392,7 +1450,7 @@ Python-level trace functions in previous versions. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or *NULL*. If + function as its first parameter, and may be any Python object, or ``NULL``. If the profile function needs to maintain state, using a different value for *obj* for each thread provides a convenient and thread-safe place to store it. The profile function is called for all monitored events except :const:`PyTrace_LINE` @@ -1458,7 +1516,7 @@ The Python interpreter provides low-level support for thread-local storage (TLS) which wraps the underlying native TLS implementation to support the Python-level thread local storage API (:class:`threading.local`). The CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:type:`void\*` value per +use a thread key and functions to associate a :c:type:`void*` value per thread. The GIL does *not* need to be held when calling these functions; they supply @@ -1469,8 +1527,8 @@ you need to include :file:`pythread.h` to use thread-local storage. .. note:: None of these API functions handle memory management on behalf of the - :c:type:`void\*` values. You need to allocate and deallocate them yourself. - If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these + :c:type:`void*` values. You need to allocate and deallocate them yourself. + If the :c:type:`void*` values happen to be :c:type:`PyObject*`, these functions don't do refcount operations on them either. .. _thread-specific-storage-api: @@ -1515,7 +1573,7 @@ is not possible due to its implementation being opaque at build time. .. c:function:: Py_tss_t* PyThread_tss_alloc() Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or *NULL* in the case of dynamic allocation + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation failure. @@ -1534,7 +1592,7 @@ is not possible due to its implementation being opaque at build time. Methods ~~~~~~~ -The parameter *key* of these functions must not be *NULL*. Moreover, the +The parameter *key* of these functions must not be ``NULL``. Moreover, the behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are undefined if the given :c:type:`Py_tss_t` has not been initialized by :c:func:`PyThread_tss_create`. @@ -1566,15 +1624,15 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by .. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - Return a zero value to indicate successfully associating a :c:type:`void\*` + Return a zero value to indicate successfully associating a :c:type:`void*` value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:type:`void\*` value. + mapping of the key to a :c:type:`void*` value. .. c:function:: void* PyThread_tss_get(Py_tss_t *key) - Return the :c:type:`void\*` value associated with a TSS key in the current - thread. This returns *NULL* if no value is associated with the key in the + Return the :c:type:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the current thread. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0d94e6b8f27b57..ff4ccb8dbbb5fc 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -25,6 +25,7 @@ Functions: * :c:func:`PyConfig_SetBytesArgv` * :c:func:`PyConfig_SetBytesString` * :c:func:`PyConfig_SetString` +* :c:func:`PyConfig_SetWideStringList` * :c:func:`PyPreConfig_InitIsolatedConfig` * :c:func:`PyPreConfig_InitPythonConfig` * :c:func:`PyStatus_Error` @@ -47,6 +48,8 @@ The preconfiguration (``PyPreConfig`` type) is stored in ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in ``PyInterpreterState.config``. +See also :ref:`Initialization, Finalization, and Threads `. + .. seealso:: :pep:`587` "Python Initialization Configuration". @@ -58,8 +61,8 @@ PyWideStringList List of ``wchar_t*`` strings. - If *length* is non-zero, *items* must be non-NULL and all strings must be - non-NULL. + If *length* is non-zero, *items* must be non-``NULL`` and all strings must be + non-``NULL``. Methods: @@ -71,8 +74,12 @@ PyWideStringList .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) - Insert *item* into *list* at *index*. If *index* is greater than *list* - length, just append *item* to *list*. + Insert *item* into *list* at *index*. + + If *index* is greater than or equal to *list* length, append *item* to + *list*. + + *index* must be greater than or equal to 0. Python must be preinitialized to call this function. @@ -189,12 +196,12 @@ PyPreConfig Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration `. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration `. @@ -234,6 +241,7 @@ PyPreConfig locale to decide if it should be coerced. .. c:member:: int coerce_c_locale_warn + If non-zero, emit a warning if the C locale is coerced. .. c:member:: int dev_mode @@ -293,7 +301,7 @@ For :ref:`Python Configuration ` (:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with command line arguments, the command line arguments must also be passed to preinitialize Python, since they have an effect on the pre-configuration -like encodings. For example, the :option:`-X` ``utf8`` command line option +like encodings. For example, the :option:`-X utf8 <-X>` command line option enables the UTF-8 Mode. ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and @@ -308,12 +316,13 @@ the preinitialization. Example using the preinitialization to enable the UTF-8 Mode:: + PyStatus status; PyPreConfig preconfig; PyPreConfig_InitPythonConfig(&preconfig); preconfig.utf8_mode = 1; - PyStatus status = Py_PreInitialize(&preconfig); + status = Py_PreInitialize(&preconfig); if (PyStatus_Exception(status)) { Py_ExitStatusException(status); } @@ -334,12 +343,12 @@ PyConfig Structure methods: - .. c:function:: PyStatus PyConfig_InitPythonConfig(PyConfig *config) + .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) Initialize configuration with :ref:`Python Configuration `. - .. c:function:: PyStatus PyConfig_InitIsolatedConfig(PyConfig *config) + .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) Initialize configuration with :ref:`Isolated Configuration `. @@ -368,6 +377,12 @@ PyConfig Preinitialize Python if needed. + .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) + + Set the list of wide strings *list* to *length* and *items*. + + Preinitialize Python if needed. + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) Read all Python configuration. @@ -412,6 +427,11 @@ PyConfig :data:`sys.base_exec_prefix`. + .. c:member:: wchar_t* base_executable + + :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment + variable value, or copy of :c:member:`PyConfig.executable`. + .. c:member:: wchar_t* base_prefix :data:`sys.base_prefix`. @@ -446,7 +466,7 @@ PyConfig .. c:member:: int dev_mode - Development mode: see :option:`-X` ``dev``. + Development mode: see :option:`-X dev <-X>`. .. c:member:: int dump_refs @@ -464,7 +484,7 @@ PyConfig .. c:member:: int faulthandler - If non-zero, call :func:`faulthandler.enable`. + If non-zero, call :func:`faulthandler.enable` at startup. .. c:member:: wchar_t* filesystem_encoding @@ -486,6 +506,9 @@ PyConfig Python home directory. + Initialized from :envvar:`PYTHONHOME` environment variable value by + default. + .. c:member:: int import_time If non-zero, profile import time. @@ -543,7 +566,7 @@ PyConfig :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden - by the function computing the :ref:`Path Configuration + by the function calculating the :ref:`Path Configuration `. .. c:member:: int optimization_level @@ -568,9 +591,9 @@ PyConfig .. c:member:: int pathconfig_warnings - If equal to 0, suppress warnings when computing the path configuration - (Unix only, Windows does not log any warning). Otherwise, warnings are - written into ``stderr``. + If equal to 0, suppress warnings when calculating the :ref:`Path + Configuration ` (Unix only, Windows does not log any + warning). Otherwise, warnings are written into ``stderr``. .. c:member:: wchar_t* prefix @@ -578,39 +601,46 @@ PyConfig .. c:member:: wchar_t* program_name - Program name. + Program name. Used to initialize :c:member:`~PyConfig.executable`, and in + early error messages. .. c:member:: wchar_t* pycache_prefix - ``.pyc`` cache prefix. + :data:`sys.pycache_prefix`: ``.pyc`` cache prefix. + + If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. .. c:member:: int quiet Quiet mode. For example, don't display the copyright and version messages - even in interactive mode. + in interactive mode. .. c:member:: wchar_t* run_command - ``python3 -c COMMAND`` argument. + ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: wchar_t* run_filename - ``python3 FILENAME`` argument. + ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: wchar_t* run_module - ``python3 -m MODULE`` argument. + ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`. .. c:member:: int show_alloc_count Show allocation counts at exit? + Set to 1 by :option:`-X showalloccount <-X>` command line option. + Need a special Python build with ``COUNT_ALLOCS`` macro defined. .. c:member:: int show_ref_count Show total reference count at exit? + Set to 1 by :option:`-X showrefcount <-X>` command line option. + Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). .. c:member:: int site_import @@ -629,7 +659,7 @@ PyConfig .. c:member:: int tracemalloc - If non-zero, call :func:`tracemalloc.start`. + If non-zero, call :func:`tracemalloc.start` at startup. .. c:member:: int use_environment @@ -645,12 +675,21 @@ PyConfig .. c:member:: PyWideStringList warnoptions - Options of the :mod:`warnings` module to build warnings filters. + :data:`sys.warnoptions`: options of the :mod:`warnings` module to build + warnings filters: lowest to highest priority. + + The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse + order: the last :c:member:`PyConfig.warnoptions` item becomes the first + item of :data:`warnings.filters` which is checked first (highest + priority). .. c:member:: int write_bytecode If non-zero, write ``.pyc`` files. + :data:`sys.dont_write_bytecode` is initialized to the inverted value of + :c:member:`~PyConfig.write_bytecode`. + .. c:member:: PyWideStringList xoptions :data:`sys._xoptions`. @@ -676,8 +715,8 @@ Function to initialize Python: The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or -``PyImport_ExtendInittab()`` is used: they must be set or called after Python +If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or +``PyImport_ExtendInittab()`` are used, they must be set or called after Python preinitialization and before the Python initialization. Example setting the program name:: @@ -685,12 +724,9 @@ Example setting the program name:: void init_python(void) { PyStatus status; - PyConfig config; - status = PyConfig_InitPythonConfig(&config); - if (PyStatus_Exception(status)) { - goto fail; - } + PyConfig config; + PyConfig_InitPythonConfig(&config); /* Set the program name. Implicitly preinitialize Python. */ status = PyConfig_SetString(&config, &config.program_name, @@ -717,14 +753,11 @@ configuration, and then override some parameters:: PyStatus init_python(const char *program_name) { PyStatus status; - PyConfig config; - status = PyConfig_InitPythonConfig(&config); - if (PyStatus_Exception(status)) { - goto done; - } + PyConfig config; + PyConfig_InitPythonConfig(&config); - /* Set the program name before reading the configuraton + /* Set the program name before reading the configuration (decode byte string from the locale encoding). Implicitly preinitialize Python. */ @@ -742,7 +775,7 @@ configuration, and then override some parameters:: /* Append our custom search path to sys.path */ status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/more/modules"); + L"/path/to/more/modules"); if (PyStatus_Exception(status)) { goto done; } @@ -773,9 +806,9 @@ isolate Python from the system. For example, to embed Python into an application. This configuration ignores global configuration variables, environments -variables and command line arguments (:c:member:`PyConfig.argv` is not parsed). -The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left -unchanged by default. +variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) +and user site directory. The C standard streams (ex: ``stdout``) and the +LC_CTYPE locale are left unchanged. Signal handlers are not installed. Configuration files are still used with this configuration. Set the :ref:`Path Configuration ` ("output fields") to ignore these @@ -803,14 +836,10 @@ Example of customized Python always running in isolated mode:: int main(int argc, char **argv) { - PyConfig config; PyStatus status; - status = PyConfig_InitPythonConfig(&config); - if (PyStatus_Exception(status)) { - goto fail; - } - + PyConfig config; + PyConfig_InitPythonConfig(&config); config.isolated = 1; /* Decode command line arguments. @@ -846,27 +875,38 @@ Path Configuration :c:type:`PyConfig` contains multiple fields for the path configuration: -* Path configuration input fields: +* Path configuration inputs: * :c:member:`PyConfig.home` - * :c:member:`PyConfig.pythonpath_env` * :c:member:`PyConfig.pathconfig_warnings` + * :c:member:`PyConfig.program_name` + * :c:member:`PyConfig.pythonpath_env` + * current working directory: to get absolute paths + * ``PATH`` environment variable to get the program full path + (from :c:member:`PyConfig.program_name`) + * ``__PYVENV_LAUNCHER__`` environment variable + * (Windows only) Application paths in the registry under + "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and + HKEY_LOCAL_MACHINE (where X.Y is the Python version). * Path configuration output fields: + * :c:member:`PyConfig.base_exec_prefix` + * :c:member:`PyConfig.base_executable` + * :c:member:`PyConfig.base_prefix` * :c:member:`PyConfig.exec_prefix` * :c:member:`PyConfig.executable` - * :c:member:`PyConfig.prefix` * :c:member:`PyConfig.module_search_paths_set`, :c:member:`PyConfig.module_search_paths` + * :c:member:`PyConfig.prefix` -If at least one "output field" is not set, Python computes the path +If at least one "output field" is not set, Python calculates the path configuration to fill unset fields. If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, -:c:member:`~PyConfig.module_search_paths` is overriden and +:c:member:`~PyConfig.module_search_paths` is overridden and :c:member:`~PyConfig.module_search_paths_set` is set to 1. -It is possible to completely ignore the function computing the default +It is possible to completely ignore the function calculating the default path configuration by setting explicitly all path configuration output fields listed above. A string is considered as set even if it is non-empty. ``module_search_paths`` is considered as set if @@ -874,7 +914,7 @@ fields listed above. A string is considered as set even if it is non-empty. configuration input fields are ignored as well. Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when -computing the path configuration (Unix only, Windows does not log any warning). +calculating the path configuration (Unix only, Windows does not log any warning). If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` @@ -905,6 +945,9 @@ The following configuration files are used by the path configuration: * ``python._pth`` (Windows only) * ``pybuilddir.txt`` (Unix only) +The ``__PYVENV_LAUNCHER__`` environment variable is used to set +:c:member:`PyConfig.base_executable` + Py_RunMain() ------------ @@ -938,7 +981,7 @@ initialization, the core feature of the :pep:`432`: * Builtin exceptions; * Builtin and frozen modules; * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet); + (ex: :data:`sys.path` doesn't exist yet). * "Main" initialization phase, Python is fully initialized: @@ -964,9 +1007,9 @@ No module is imported during the "Core" phase and the ``importlib`` module is not configured: the :ref:`Path Configuration ` is only applied during the "Main" phase. It may allow to customize Python in Python to override or tune the :ref:`Path Configuration `, maybe -install a custom sys.meta_path importer or an import hook, etc. +install a custom :data:`sys.meta_path` importer or an import hook, etc. -It may become possible to compute the :ref:`Path Configuration +It may become possible to calculatin the :ref:`Path Configuration ` in Python, after the Core phase and before the Main phase, which is one of the :pep:`432` motivation. @@ -981,14 +1024,9 @@ phases:: void init_python(void) { PyStatus status; - PyConfig config; - - status = PyConfig_InitPythonConfig(&config); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } + PyConfig config; + PyConfig_InitPythonConfig(&config); config._init_main = 0; /* ... customize 'config' configuration ... */ diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index a1c8d34a7ea02f..fd1268e310260d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -69,10 +69,12 @@ standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning with ``_Py`` are for internal use by the Python implementation and should not be used by extension writers. Structure member names do not have a reserved prefix. -**Important:** user code should never define names that begin with ``Py`` or -``_Py``. This confuses the reader, and jeopardizes the portability of the user -code to future Python versions, which may define additional names beginning with -one of these prefixes. +.. note:: + + User code should never define names that begin with ``Py`` or ``_Py``. This + confuses the reader, and jeopardizes the portability of the user code to + future Python versions, which may define additional names beginning with one + of these prefixes. The header files are typically installed with Python. On Unix, these are located in the directories :file:`{prefix}/include/pythonversion/` and @@ -90,9 +92,9 @@ multi-platform builds since the platform independent headers under :envvar:`prefix` include the platform specific headers from :envvar:`exec_prefix`. -C++ users should note that though the API is defined entirely using C, the -header files do properly declare the entry points to be ``extern "C"``, so there -is no need to do anything special to use the API from C++. +C++ users should note that although the API is defined entirely using C, the +header files properly declare the entry points to be ``extern "C"``. As a result, +there is no need to do anything special to use the API from C++. Useful macros @@ -150,7 +152,7 @@ complete listing. .. c:macro:: Py_GETENV(s) - Like ``getenv(s)``, but returns *NULL* if :option:`-E` was passed on the + Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set). .. c:macro:: Py_UNUSED(arg) @@ -172,6 +174,39 @@ complete listing. .. versionchanged:: 3.8 MSVC support was added. +.. c:macro:: PyDoc_STRVAR(name, str) + + Creates a variable with name ``name`` that can be used in docstrings. + If Python is built without docstrings, the value will be empty. + + Use :c:macro:`PyDoc_STRVAR` for docstrings to support building + Python without docstrings, as specified in :pep:`7`. + + Example:: + + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } + +.. c:macro:: PyDoc_STR(str) + + Creates a docstring for the given input string or an empty string + if docstrings are disabled. + + Use :c:macro:`PyDoc_STR` in specifying docstrings to support + building Python without docstrings, as specified in :pep:`7`. + + Example:: + + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; .. _api-objects: @@ -181,13 +216,13 @@ Objects, Types and Reference Counts .. index:: object: type Most Python/C API functions have one or more arguments as well as a return value -of type :c:type:`PyObject\*`. This type is a pointer to an opaque data type +of type :c:type:`PyObject*`. This type is a pointer to an opaque data type representing an arbitrary Python object. Since all Python object types are treated the same way by the Python language in most situations (e.g., assignments, scope rules, and argument passing), it is only fitting that they should be represented by a single C type. Almost all Python objects live on the heap: you never declare an automatic or static variable of type -:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject\*` can be +:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject*` can be declared. The sole exception are the type objects; since these must never be deallocated, they are typically static :c:type:`PyTypeObject` objects. @@ -448,7 +483,7 @@ Types There are few other data types that play a significant role in the Python/C API; most are simple C types such as :c:type:`int`, :c:type:`long`, -:c:type:`double` and :c:type:`char\*`. A few structure types are used to +:c:type:`double` and :c:type:`char*`. A few structure types are used to describe static tables used to list the functions exported by a module or the data attributes of a new object type, and another is used to describe the value of a complex number. These will be discussed together with the functions that @@ -473,7 +508,7 @@ functions in the Python/C API can raise exceptions, unless an explicit claim is made otherwise in a function's documentation. In general, when a function encounters an error, it sets an exception, discards any object references that it owns, and returns an error indicator. If not documented otherwise, this -indicator is either *NULL* or ``-1``, depending on the function's return type. +indicator is either ``NULL`` or ``-1``, depending on the function's return type. A few functions return a Boolean true/false result, with false indicating an error. Very few functions return no explicit error indicator or have an ambiguous return value, and require explicit testing for errors with @@ -488,13 +523,13 @@ using global storage in an unthreaded application). A thread can be in one of two states: an exception has occurred, or not. The function :c:func:`PyErr_Occurred` can be used to check for this: it returns a borrowed reference to the exception type object when an exception has occurred, and -*NULL* otherwise. There are a number of functions to set the exception state: +``NULL`` otherwise. There are a number of functions to set the exception state: :c:func:`PyErr_SetString` is the most common (though not the most general) function to set the exception state, and :c:func:`PyErr_Clear` clears the exception state. The full exception state consists of three objects (all of which can be -*NULL*): the exception type, the corresponding exception value, and the +``NULL``): the exception type, the corresponding exception value, and the traceback. These have the same meanings as the Python result of ``sys.exc_info()``; however, they are not the same: the Python objects represent the last exception being handled by a Python :keyword:`try` ... @@ -595,10 +630,10 @@ Here is the corresponding C code, in all its glory:: This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and :c:func:`PyErr_Clear` to handle specific exceptions, and the use of -:c:func:`Py_XDECREF` to dispose of owned references that may be *NULL* (note the +:c:func:`Py_XDECREF` to dispose of owned references that may be ``NULL`` (note the ``'X'`` in the name; :c:func:`Py_DECREF` would crash when confronted with a -*NULL* reference). It is important that the variables used to hold owned -references are initialized to *NULL* for this to work; likewise, the proposed +``NULL`` reference). It is important that the variables used to hold owned +references are initialized to ``NULL`` for this to work; likewise, the proposed return value is initialized to ``-1`` (failure) and only set to success after the final call made is successful. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 546efb518a7af5..a2992b3452f91c 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -16,8 +16,8 @@ There are two functions specifically for working with iterators. Return the next value from the iteration *o*. The object must be an iterator (it is up to the caller to check this). If there are no remaining values, - returns *NULL* with no exception set. If an error occurs while retrieving - the item, returns *NULL* and passes along the exception. + returns ``NULL`` with no exception set. If an error occurs while retrieving + the item, returns ``NULL`` and passes along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: @@ -29,7 +29,7 @@ something like this:: /* propagate error */ } - while (item = PyIter_Next(iterator)) { + while ((item = PyIter_Next(iterator))) { /* do something with item */ ... /* release reference when done */ diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 279783a243a108..b247cdfba0187e 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -33,7 +33,7 @@ List Objects .. c:function:: PyObject* PyList_New(Py_ssize_t len) - Return a new list of length *len* on success, or *NULL* on failure. + Return a new list of length *len* on success, or ``NULL`` on failure. .. note:: @@ -59,9 +59,9 @@ List Objects .. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The - position must be positive, indexing from the end of the list is not - supported. If *index* is out of bounds, return *NULL* and set an - :exc:`IndexError` exception. + position must be non-negative; indexing from the end of the list is not + supported. If *index* is out of bounds (<0 or >=len(list)), + return ``NULL`` and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) @@ -71,8 +71,9 @@ List Objects .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) - Set the item at index *index* in list to *item*. Return ``0`` on success - or ``-1`` on failure. + Set the item at index *index* in list to *item*. Return ``0`` on success. + If *index* is out of bounds, return ``-1`` and set an :exc:`IndexError` + exception. .. note:: @@ -110,18 +111,17 @@ List Objects .. c:function:: PyObject* PyList_GetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high) Return a list of the objects in *list* containing the objects *between* *low* - and *high*. Return *NULL* and set an exception if unsuccessful. Analogous - to ``list[low:high]``. Negative indices, as when slicing from Python, are not - supported. + and *high*. Return ``NULL`` and set an exception if unsuccessful. Analogous + to ``list[low:high]``. Indexing from the end of the list is not supported. .. c:function:: int PyList_SetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high, PyObject *itemlist) Set the slice of *list* between *low* and *high* to the contents of *itemlist*. Analogous to ``list[low:high] = itemlist``. The *itemlist* may - be *NULL*, indicating the assignment of an empty list (slice deletion). - Return ``0`` on success, ``-1`` on failure. Negative indices, as when - slicing from Python, are not supported. + be ``NULL``, indicating the assignment of an empty list (slice deletion). + Return ``0`` on success, ``-1`` on failure. Indexing from the end of the + list is not supported. .. c:function:: int PyList_Sort(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 6be29f9cd32eaa..3bada415aa37ef 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -38,7 +38,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromLong(long v) - Return a new :c:type:`PyLongObject` object from *v*, or *NULL* on failure. + Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. The current implementation keeps an array of integer objects for all integers between ``-5`` and ``256``, when you create an int in that range you actually @@ -50,43 +50,43 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromSsize_t(Py_ssize_t v) Return a new :c:type:`PyLongObject` object from a C :c:type:`Py_ssize_t`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromSize_t(size_t v) Return a new :c:type:`PyLongObject` object from a C :c:type:`size_t`, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromLongLong(long long v) - Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or *NULL* + Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long long`, - or *NULL* on failure. + or ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromDouble(double v) Return a new :c:type:`PyLongObject` object from the integer part of *v*, or - *NULL* on failure. + ``NULL`` on failure. .. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) Return a new :c:type:`PyLongObject` based on the string value in *str*, which - is interpreted according to the radix in *base*. If *pend* is non-*NULL*, + is interpreted according to the radix in *base*. If *pend* is non-``NULL``, *\*pend* will point to the first character in *str* which follows the representation of the number. If *base* is ``0``, *str* is interpreted using the :ref:`integers` definition; in this case, leading zeros in a @@ -98,11 +98,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) - Convert a sequence of Unicode digits to a Python integer value. The Unicode - string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` - and then converted using :c:func:`PyLong_FromString`. + Convert a sequence of Unicode digits to a Python integer value. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.10 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyLong_FromUnicodeObject`. @@ -110,9 +108,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) Convert a sequence of Unicode digits in the string *u* to a Python integer - value. The Unicode string is first encoded to a byte string using - :c:func:`PyUnicode_EncodeDecimal` and then converted using - :c:func:`PyLong_FromString`. + value. .. versionadded:: 3.3 @@ -179,7 +175,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. + :c:type:`long long`. Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. @@ -288,7 +284,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to + disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. @@ -307,7 +304,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` + to disambiguate. .. versionchanged:: 3.8 Use :meth:`__index__` if available. @@ -334,4 +332,4 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. is only assured to produce a usable :c:type:`void` pointer for values created with :c:func:`PyLong_FromVoidPtr`. - Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 4244b47af75f1a..682160d1475c1c 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -14,8 +14,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Return ``1`` if the object provides mapping protocol or supports slicing, and ``0`` otherwise. Note that it returns ``1`` for Python classes with a :meth:`__getitem__` method since in general case it is impossible to - determine what the type of keys it supports. This function always - succeeds. + determine what type of keys it supports. This function always succeeds. .. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) @@ -29,7 +28,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - Return element of *o* corresponding to the string *key* or *NULL* on failure. + Return element of *o* corresponding to the string *key* or ``NULL`` on failure. This is the equivalent of the Python expression ``o[key]``. See also :c:func:`PyObject_GetItem`. @@ -38,7 +37,8 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Map the string *key* to the value *v* in object *o*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. + See also :c:func:`PyObject_SetItem`. This function *does not* steal a + reference to *v*. .. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) @@ -79,7 +79,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Keys(PyObject *o) On success, return a list of the keys in object *o*. On failure, return - *NULL*. + ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. @@ -88,7 +88,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Values(PyObject *o) On success, return a list of the values in object *o*. On failure, return - *NULL*. + ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. @@ -97,7 +97,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_Items(PyObject *o) On success, return a list of the items in object *o*, where each item is a - tuple containing a key-value pair. On failure, return *NULL*. + tuple containing a key-value pair. On failure, return ``NULL``. .. versionchanged:: 3.7 Previously, the function returned a list or a tuple. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index b086830feb14f1..027ffd5812413e 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -16,7 +16,7 @@ Numeric values are stored with the least significant byte first. The module supports two versions of the data format: version 0 is the historical version, version 1 shares interned strings in the file, and upon unmarshalling. Version 2 uses a binary format for floating point numbers. -*Py_MARSHAL_VERSION* indicates the current file format (currently 2). +``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) @@ -25,12 +25,16 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. the least-significant 32 bits of *value*; regardless of the size of the native :c:type:`long` type. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) Marshal a Python object, *value*, to *file*. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) @@ -43,7 +47,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - Return a C :c:type:`long` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`long` from the data stream in a :c:type:`FILE*` opened for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:type:`long`. @@ -53,7 +57,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - Return a C :c:type:`short` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`short` from the data stream in a :c:type:`FILE*` opened for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:type:`short`. @@ -63,16 +67,16 @@ The following functions allow marshalled values to be read back in. .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can @@ -81,7 +85,7 @@ The following functions allow marshalled values to be read back in. anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. .. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) @@ -90,5 +94,5 @@ The following functions allow marshalled values to be read back in. containing *len* bytes pointed to by *data*. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. + or :exc:`TypeError`) and returns ``NULL``. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index ab49e48782d71a..91b901d726e61c 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -67,7 +67,7 @@ example:: In this example, the memory request for the I/O buffer is handled by the C library allocator. The Python memory manager is involved only in the allocation -of the string object returned as a result. +of the bytes object returned as a result. In most situations, however, it is recommended to allocate memory from the Python heap specifically because the latter is under control of the Python @@ -109,10 +109,10 @@ zero bytes. .. c:function:: void* PyMem_RawMalloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -120,11 +120,11 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been + non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -135,15 +135,15 @@ zero bytes. Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyMem_RawMalloc(n)``; else if + If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or :c:func:`PyMem_RawCalloc`. - If the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* + If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -154,7 +154,7 @@ zero bytes. :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. .. _memoryinterface: @@ -180,10 +180,10 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyMem_Malloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -191,11 +191,11 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called + non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -206,14 +206,14 @@ The :ref:`default memory allocator ` uses the Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* + If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - If the request fails, :c:func:`PyMem_Realloc` returns *NULL* and *p* remains + If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -224,7 +224,7 @@ The :ref:`default memory allocator ` uses the :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. The following type-oriented macros are provided for convenience. Note that *TYPE* refers to any C type. @@ -233,15 +233,15 @@ The following type-oriented macros are provided for convenience. Note that .. c:function:: TYPE* PyMem_New(TYPE, size_t n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE\*`. The memory will not have + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have been initialized in any way. .. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE\*`. On return, - *p* will be a pointer to the new memory area, or *NULL* in the event of + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, + *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. This is a C preprocessor macro; *p* is always reassigned. Save the original @@ -282,10 +282,10 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as if ``PyObject_Malloc(1)`` had been called instead. The memory will not have been initialized in any way. @@ -293,11 +293,11 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called instead. .. versionadded:: 3.5 @@ -308,14 +308,14 @@ The :ref:`default object allocator ` uses the Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. - If *p* is *NULL*, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. + returned pointer is non-``NULL``. - Unless *p* is *NULL*, it must have been returned by a previous call to + Unless *p* is ``NULL``, it must have been returned by a previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - If the request fails, :c:func:`PyObject_Realloc` returns *NULL* and *p* remains + If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains a valid pointer to the previous memory area. @@ -326,7 +326,7 @@ The :ref:`default object allocator ` uses the :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called before, undefined behavior occurs. - If *p* is *NULL*, no operation is performed. + If *p* is ``NULL``, no operation is performed. .. _default-memory-allocators: @@ -388,7 +388,7 @@ Customize Memory Allocators Enum used to identify an allocator domain. Domains: - .. c:var:: PYMEM_DOMAIN_RAW + .. c:macro:: PYMEM_DOMAIN_RAW Functions: @@ -397,7 +397,7 @@ Customize Memory Allocators * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` - .. c:var:: PYMEM_DOMAIN_MEM + .. c:macro:: PYMEM_DOMAIN_MEM Functions: @@ -406,7 +406,7 @@ Customize Memory Allocators * :c:func:`PyMem_Calloc` * :c:func:`PyMem_Free` - .. c:var:: PYMEM_DOMAIN_OBJ + .. c:macro:: PYMEM_DOMAIN_OBJ Functions: @@ -424,7 +424,7 @@ Customize Memory Allocators Set the memory block allocator of the specified domain. - The new allocator must return a distinct non-NULL pointer when requesting + The new allocator must return a distinct non-``NULL`` pointer when requesting zero bytes. For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be @@ -472,7 +472,7 @@ Customize Memory Allocators if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. - .. versionchanged:: 3.8.0 + .. versionchanged:: 3.8 Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` @@ -516,14 +516,14 @@ Customize pymalloc Arena Allocator +--------------------------------------------------+---------------------------------------+ | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + | ``void free(void *ctx, void *ptr, size_t size)`` | free an arena | +--------------------------------------------------+---------------------------------------+ -.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) Get the arena allocator. -.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) Set the arena allocator. @@ -533,7 +533,7 @@ tracemalloc C API .. versionadded:: 3.7 -.. c:function: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) +.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) Track an allocated memory block in the :mod:`tracemalloc` module. @@ -542,7 +542,7 @@ tracemalloc C API If memory block is already tracked, update the existing trace. -.. c:function: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) +.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) Untrack an allocated memory block in the :mod:`tracemalloc` module. Do nothing if the block was not tracked. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 77afb020405a74..de429e5c11dc76 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -57,7 +57,7 @@ any other object. .. c:function:: Py_buffer *PyMemoryView_GET_BASE(PyObject *mview) Return either a pointer to the exporting object that the memoryview is based - on or *NULL* if the memoryview has been created by one of the functions + on or ``NULL`` if the memoryview has been created by one of the functions :c:func:`PyMemoryView_FromMemory` or :c:func:`PyMemoryView_FromBuffer`. *mview* **must** be a memoryview instance. diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 1ad805e269aa2d..b1862d796c9f41 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -21,7 +21,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call .. c:function:: int PyInstanceMethod_Check(PyObject *o) Return true if *o* is an instance method object (has type - :c:data:`PyInstanceMethod_Type`). The parameter must not be *NULL*. + :c:data:`PyInstanceMethod_Type`). The parameter must not be ``NULL``. .. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) @@ -64,14 +64,14 @@ no longer available. .. c:function:: int PyMethod_Check(PyObject *o) Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be *NULL*. + parameter must not be ``NULL``. .. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) Return a new method object, with *func* being any callable object and *self* the instance the method should be bound. *func* is the function that will - be called when the method is called. *self* must not be *NULL*. + be called when the method is called. *self* must not be ``NULL``. .. c:function:: PyObject* PyMethod_Function(PyObject *meth) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 68cbda2938f3f0..f840dd90558cce 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -61,7 +61,7 @@ Module Objects Return the dictionary object that implements *module*'s namespace; this object is the same as the :attr:`~object.__dict__` attribute of the module object. If *module* is not a module object (or a subtype of a module object), - :exc:`SystemError` is raised and *NULL* is returned. + :exc:`SystemError` is raised and ``NULL`` is returned. It is recommended extensions use other :c:func:`PyModule_\*` and :c:func:`PyObject_\*` functions rather than directly manipulate a module's @@ -75,7 +75,7 @@ Module Objects single: SystemError (built-in exception) Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned. + or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. .. versionadded:: 3.3 @@ -88,14 +88,14 @@ Module Objects .. c:function:: void* PyModule_GetState(PyObject *module) Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or *NULL*. See + allocated at module creation time, or ``NULL``. See :c:member:`PyModuleDef.m_size`. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was - created, or *NULL* if the module wasn't created from a definition. + created, or ``NULL`` if the module wasn't created from a definition. .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -106,7 +106,7 @@ Module Objects Return the name of the file from which *module* was loaded using *module*'s :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return + unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 @@ -153,7 +153,7 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: const char *m_doc Docstring for the module; usually a docstring variable created with - :c:func:`PyDoc_STRVAR` is used. + :c:macro:`PyDoc_STRVAR` is used. .. c:member:: Py_ssize_t m_size @@ -178,17 +178,17 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: PyMethodDef* m_methods A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present. + :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. .. c:member:: PyModuleDef_Slot* m_slots An array of slot definitions for multi-phase initialization, terminated by a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be *NULL*. + When using single-phase initialization, *m_slots* must be ``NULL``. .. versionchanged:: 3.5 - Prior to version 3.5, this member was always set to *NULL*, + Prior to version 3.5, this member was always set to ``NULL``, and was defined as: .. c:member:: inquiry m_reload @@ -196,20 +196,20 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. This function may be called before module state + ``NULL`` if not needed. This function may be called before module state is allocated (:c:func:`PyModule_GetState()` may return `NULL`), and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. This function may be called before module state + ``NULL`` if not needed. This function may be called before module state is allocated (:c:func:`PyModule_GetState()` may return `NULL`), and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free - A function to call during deallocation of the module object, or *NULL* if + A function to call during deallocation of the module object, or ``NULL`` if not needed. This function may be called before module state is allocated (:c:func:`PyModule_GetState()` may return `NULL`), and before the :c:member:`Py_mod_exec` function is executed. @@ -278,7 +278,7 @@ instance must be initialized with the following function: Ensures a module definition is a properly initialized Python object that correctly reports its type and reference count. - Returns *def* cast to ``PyObject*``, or *NULL* if an error occurred. + Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. .. versionadded:: 3.5 @@ -301,7 +301,7 @@ The *m_slots* array must be terminated by a slot with id 0. The available slot types are: -.. c:var:: Py_mod_create +.. c:macro:: Py_mod_create Specifies a function that is called to create the module object itself. The *value* pointer of this slot must point to a function of the signature: @@ -311,7 +311,7 @@ The available slot types are: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. It should return a new module object, or set an error - and return *NULL*. + and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may @@ -330,10 +330,10 @@ The available slot types are: :c:type:`PyModule_Type`. Any type can be used, as long as it supports setting and getting import-related attributes. However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-*NULL* ``m_traverse``, ``m_clear``, + ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. -.. c:var:: Py_mod_exec +.. c:macro:: Py_mod_exec Specifies a function that is called to *execute* the module. This is equivalent to executing the code of a Python module: typically, @@ -394,7 +394,7 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) - Add the functions from the *NULL* terminated *functions* array to *module*. + Add the functions from the ``NULL`` terminated *functions* array to *module*. Refer to the :c:type:`PyMethodDef` documentation for details on individual entries (due to the lack of a shared module namespace, module level "functions" implemented in C typically receive the module as their first @@ -417,7 +417,22 @@ state: Add an object to *module* as *name*. This is a convenience function which can be used from the module's initialization function. This steals a reference to - *value*. Return ``-1`` on error, ``0`` on success. + *value* on success. Return ``-1`` on error, ``0`` on success. + + .. note:: + + Unlike other functions that steal references, ``PyModule_AddObject()`` only + decrements the reference count of *value* **on success**. + + This means that its return value must be checked, and calling code must + :c:func:`Py_DECREF` *value* manually on error. Example usage:: + + Py_INCREF(spam); + if (PyModule_AddObject(module, "spam", spam) < 0) { + Py_DECREF(module); + Py_DECREF(spam); + return NULL; + } .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -430,7 +445,7 @@ state: Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - *NULL*-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. .. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) @@ -461,7 +476,7 @@ since multiple such modules can be created from a single definition. Returns the module object that was created from *def* for the current interpreter. This method requires that the module object has been attached to the interpreter state with :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not - found or has not been attached to the interpreter state yet, it returns *NULL*. + found or has not been attached to the interpreter state yet, it returns ``NULL``. .. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def) @@ -470,10 +485,21 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. + Python calls ``PyState_AddModule`` automatically after importing a module, + so it is unnecessary (but harmless) to call it from module initialization + code. An explicit call is needed only if the module's own init code + subsequently calls ``PyState_FindModule``. + The function is mainly intended for implementing alternative import + mechanisms (either by calling it directly, or by referring to its + implementation for details of the required state updates). + + Return 0 on success or -1 on failure. + .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. + Return 0 on success or -1 on failure. .. versionadded:: 3.3 diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 74932f6e7445f0..620204ca8e2295 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -17,25 +17,25 @@ Number Protocol .. c:function:: PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) - Returns the result of adding *o1* and *o2*, or *NULL* on failure. This is the + Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 + o2``. .. c:function:: PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. This is + Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 - o2``. .. c:function:: PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. This is + Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 * o2``. .. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2) - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on + Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 @ o2``. .. versionadded:: 3.5 @@ -43,14 +43,14 @@ Number Protocol .. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) - Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is + Return the floor of *o1* divided by *o2*, or ``NULL`` on failure. This is equivalent to the "classic" division of integers. .. c:function:: PyObject* PyNumber_TrueDivide(PyObject *o1, PyObject *o2) Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary + *o2*, or ``NULL`` on failure. The return value is "approximate" because binary floating point numbers are approximate; it is not possible to represent all real numbers in base two. This function can return a floating point value when passed two integers. @@ -58,7 +58,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_Remainder(PyObject *o1, PyObject *o2) - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. This is + Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 % o2``. @@ -66,7 +66,7 @@ Number Protocol .. index:: builtin: divmod - See the built-in function :func:`divmod`. Returns *NULL* on failure. This is + See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``divmod(o1, o2)``. @@ -74,21 +74,21 @@ Number Protocol .. index:: builtin: pow - See the built-in function :func:`pow`. Returns *NULL* on failure. This is the + See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. - If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing *NULL* for + If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing ``NULL`` for *o3* would cause an illegal memory access). .. c:function:: PyObject* PyNumber_Negative(PyObject *o) - Returns the negation of *o* on success, or *NULL* on failure. This is the + Returns the negation of *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``-o``. .. c:function:: PyObject* PyNumber_Positive(PyObject *o) - Returns *o* on success, or *NULL* on failure. This is the equivalent of the + Returns *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``+o``. @@ -96,70 +96,70 @@ Number Protocol .. index:: builtin: abs - Returns the absolute value of *o*, or *NULL* on failure. This is the equivalent + Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``abs(o)``. .. c:function:: PyObject* PyNumber_Invert(PyObject *o) - Returns the bitwise negation of *o* on success, or *NULL* on failure. This is + Returns the bitwise negation of *o* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``~o``. .. c:function:: PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 << o2``. .. c:function:: PyObject* PyNumber_Rshift(PyObject *o1, PyObject *o2) - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 >> o2``. .. c:function:: PyObject* PyNumber_And(PyObject *o1, PyObject *o2) - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. + Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. This is the equivalent of the Python expression ``o1 & o2``. .. c:function:: PyObject* PyNumber_Xor(PyObject *o1, PyObject *o2) - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 ^ o2``. .. c:function:: PyObject* PyNumber_Or(PyObject *o1, PyObject *o2) - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. + Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o1 | o2``. .. c:function:: PyObject* PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2) - Returns the result of adding *o1* and *o2*, or *NULL* on failure. The operation + Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 += o2``. .. c:function:: PyObject* PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2) - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. The + Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 -= o2``. .. c:function:: PyObject* PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2) - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. The + Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 *= o2``. .. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2) - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on + Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 @= o2``. @@ -168,7 +168,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) - Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure. + Returns the mathematical floor of dividing *o1* by *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 //= o2``. @@ -176,7 +176,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceTrueDivide(PyObject *o1, PyObject *o2) Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary + *o2*, or ``NULL`` on failure. The return value is "approximate" because binary floating point numbers are approximate; it is not possible to represent all real numbers in base two. This function can return a floating point value when passed two integers. The operation is done *in-place* when *o1* supports it. @@ -184,7 +184,7 @@ Number Protocol .. c:function:: PyObject* PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2) - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. The + Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 %= o2``. @@ -193,44 +193,44 @@ Number Protocol .. index:: builtin: pow - See the built-in function :func:`pow`. Returns *NULL* on failure. The operation + See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 **= o2`` when o3 is :c:data:`Py_None`, or an in-place variant of ``pow(o1, o2, o3)`` otherwise. If *o3* is to be ignored, pass :c:data:`Py_None` - in its place (passing *NULL* for *o3* would cause an illegal memory access). + in its place (passing ``NULL`` for *o3* would cause an illegal memory access). .. c:function:: PyObject* PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2) - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on + Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 <<= o2``. .. c:function:: PyObject* PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2) - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on + Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 >>= o2``. .. c:function:: PyObject* PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2) - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. The + Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 &= o2``. .. c:function:: PyObject* PyNumber_InPlaceXor(PyObject *o1, PyObject *o2) - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on + Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 ^= o2``. .. c:function:: PyObject* PyNumber_InPlaceOr(PyObject *o1, PyObject *o2) - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. The + Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 |= o2``. @@ -239,7 +239,7 @@ Number Protocol .. index:: builtin: int - Returns the *o* converted to an integer object on success, or *NULL* on + Returns the *o* converted to an integer object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``int(o)``. @@ -247,13 +247,13 @@ Number Protocol .. index:: builtin: float - Returns the *o* converted to a float object on success, or *NULL* on failure. + Returns the *o* converted to a float object on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``float(o)``. .. c:function:: PyObject* PyNumber_Index(PyObject *o) - Returns the *o* converted to a Python int on success or *NULL* with a + Returns the *o* converted to a Python int on success or ``NULL`` with a :exc:`TypeError` exception raised on failure. @@ -274,9 +274,9 @@ Number Protocol If *o* can be converted to a Python int but the attempt to convert to a Py_ssize_t value would raise an :exc:`OverflowError`, then the *exc* argument is the type of exception that will be raised (usually - :exc:`IndexError` or :exc:`OverflowError`). If *exc* is *NULL*, then the - exception is cleared and the value is clipped to *PY_SSIZE_T_MIN* for a negative - integer or *PY_SSIZE_T_MAX* for a positive integer. + :exc:`IndexError` or :exc:`OverflowError`). If *exc* is ``NULL``, then the + exception is cleared and the value is clipped to ``PY_SSIZE_T_MIN`` for a negative + integer or ``PY_SSIZE_T_MAX`` for a positive integer. .. c:function:: int PyIndex_Check(PyObject *o) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index ce0d05942f4e79..d4e8b7468089c0 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -53,14 +53,14 @@ Object Protocol .. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python + value on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o.attr_name``. .. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python + value on success, or ``NULL`` on failure. This is the equivalent of the Python expression ``o.attr_name``. @@ -81,7 +81,7 @@ Object Protocol return ``0`` on success. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is + If *v* is ``NULL``, the attribute is deleted, however this feature is deprecated in favour of using :c:func:`PyObject_DelAttr`. @@ -92,7 +92,7 @@ Object Protocol return ``0`` on success. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is + If *v* is ``NULL``, the attribute is deleted, however this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. @@ -128,7 +128,7 @@ Object Protocol .. versionadded:: 3.3 -.. c:function:: int PyObject_GenericSetDict(PyObject *o, void *context) +.. c:function:: int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) A generic implementation for the setter of a ``__dict__`` descriptor. This implementation does not allow the dictionary to be deleted. @@ -143,7 +143,7 @@ Object Protocol :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of the Python expression ``o1 op o2``, where ``op`` is the operator corresponding - to *opid*. Returns the value of the comparison on success, or *NULL* on failure. + to *opid*. Returns the value of the comparison on success, or ``NULL`` on failure. .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) @@ -165,7 +165,7 @@ Object Protocol .. index:: builtin: repr Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the + representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. .. versionchanged:: 3.4 @@ -188,7 +188,7 @@ Object Protocol .. c:function:: PyObject* PyObject_Str(PyObject *o) Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the + representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. @@ -200,7 +200,7 @@ Object Protocol .. index:: builtin: bytes - Compute a bytes representation of object *o*. *NULL* is returned on + Compute a bytes representation of object *o*. ``NULL`` is returned on failure and a bytes object on success. This is equivalent to the Python expression ``bytes(o)``, when *o* is not an integer. Unlike ``bytes(o)``, a TypeError is raised when *o* is an integer instead of a zero-initialized @@ -258,10 +258,11 @@ Object Protocol Call a callable Python object *callable*, with arguments given by the tuple *args*, and named arguments given by the dictionary *kwargs*. - *args* must not be *NULL*, use an empty tuple if no arguments are needed. - If no named arguments are needed, *kwargs* can be *NULL*. + *args* must not be ``NULL``, use an empty tuple if no arguments are needed. + If no named arguments are needed, *kwargs* can be ``NULL``. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This is the equivalent of the Python expression: ``callable(*args, **kwargs)``. @@ -270,9 +271,10 @@ Object Protocol .. c:function:: PyObject* PyObject_CallObject(PyObject *callable, PyObject *args) Call a callable Python object *callable*, with arguments given by the - tuple *args*. If no arguments are needed, then *args* can be *NULL*. + tuple *args*. If no arguments are needed, then *args* can be ``NULL``. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This is the equivalent of the Python expression: ``callable(*args)``. @@ -281,13 +283,14 @@ Object Protocol Call a callable Python object *callable*, with a variable number of C arguments. The C arguments are described using a :c:func:`Py_BuildValue` style format - string. The format can be *NULL*, indicating that no arguments are provided. + string. The format can be ``NULL``, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This is the equivalent of the Python expression: ``callable(*args)``. - Note that if you only pass :c:type:`PyObject \*` args, + Note that if you only pass :c:type:`PyObject *` args, :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. .. versionchanged:: 3.4 @@ -300,39 +303,43 @@ Object Protocol arguments. The C arguments are described by a :c:func:`Py_BuildValue` format string that should produce a tuple. - The format can be *NULL*, indicating that no arguments are provided. + The format can be ``NULL``, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This is the equivalent of the Python expression: ``obj.name(arg1, arg2, ...)``. - Note that if you only pass :c:type:`PyObject \*` args, + Note that if you only pass :c:type:`PyObject *` args, :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. .. versionchanged:: 3.4 The types of *name* and *format* were changed from ``char *``. -.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) +.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ...) Call a callable Python object *callable*, with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. + :c:type:`PyObject*` arguments. The arguments are provided as a variable number + of parameters followed by ``NULL``. - Returns the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This is the equivalent of the Python expression: ``callable(arg1, arg2, ...)``. -.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ..., NULL) +.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) Calls a method of the Python object *obj*, where the name of the method is given as a Python string object in *name*. It is called with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or - *NULL* on failure. + :c:type:`PyObject*` arguments. The arguments are provided as a variable number + of parameters followed by ``NULL``. + + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. .. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -347,7 +354,7 @@ Object Protocol To get actual number of arguments, use :c:func:`PyVectorcall_NARGS(nargsf) `. - *kwnames* can be either NULL (no keyword arguments) or a tuple of keyword + *kwnames* can be either ``NULL`` (no keyword arguments) or a tuple of keyword names. In the latter case, the values of the keyword arguments are stored in *args* after the positional arguments. The number of keyword arguments does not influence *nargsf*. @@ -355,7 +362,8 @@ Object Protocol *kwnames* must contain only objects of type ``str`` (not a subclass), and all keys must be unique. - Return the result of the call on success, or *NULL* on failure. + Return the result of the call on success, or raise an exception and return + ``NULL`` on failure. This uses the vectorcall protocol if the callable supports it; otherwise, the arguments are converted to use @@ -369,7 +377,7 @@ Object Protocol .. versionadded:: 3.8 -.. c:var:: PY_VECTORCALL_ARGUMENTS_OFFSET +.. c:macro:: PY_VECTORCALL_ARGUMENTS_OFFSET If set in a vectorcall *nargsf* argument, the callee is allowed to temporarily change ``args[-1]``. In other words, *args* points to @@ -394,7 +402,7 @@ Object Protocol .. c:function:: PyObject* _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) Same as :c:func:`_PyObject_Vectorcall` except that the keyword arguments - are passed as a dictionary in *kwdict*. This may be *NULL* if there + are passed as a dictionary in *kwdict*. This may be ``NULL`` if there are no keyword arguments. For callables supporting :c:data:`vectorcall `, @@ -450,19 +458,19 @@ Object Protocol .. index:: builtin: type - When *o* is non-*NULL*, returns a type object corresponding to the object type - of object *o*. On failure, raises :exc:`SystemError` and returns *NULL*. This + When *o* is non-``NULL``, returns a type object corresponding to the object type + of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This is equivalent to the Python expression ``type(o)``. This function increments the reference count of the return value. There's really no reason to use this function instead of the common expression ``o->ob_type``, which returns a - pointer of type :c:type:`PyTypeObject\*`, except when the incremented reference + pointer of type :c:type:`PyTypeObject*`, except when the incremented reference count is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) Return true if the object *o* is of type *type* or a subtype of *type*. Both - parameters must be non-*NULL*. + parameters must be non-``NULL``. .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) @@ -487,7 +495,7 @@ Object Protocol .. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) - Return element of *o* corresponding to the object *key* or *NULL* on failure. + Return element of *o* corresponding to the object *key* or ``NULL`` on failure. This is the equivalent of the Python expression ``o[key]``. @@ -495,7 +503,8 @@ Object Protocol Map the object *key* to the value *v*. Raise an exception and return ``-1`` on failure; return ``0`` on success. This is the - equivalent of the Python statement ``o[key] = v``. + equivalent of the Python statement ``o[key] = v``. This function *does + not* steal a reference to *v*. .. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key) @@ -507,15 +516,15 @@ Object Protocol .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly - empty) list of strings appropriate for the object argument, or *NULL* if there - was an error. If the argument is *NULL*, this is like the Python ``dir()``, + empty) list of strings appropriate for the object argument, or ``NULL`` if there + was an error. If the argument is ``NULL``, this is like the Python ``dir()``, returning the names of the current locals; in this case, if no execution frame - is active then *NULL* is returned but :c:func:`PyErr_Occurred` will return false. + is active then ``NULL`` is returned but :c:func:`PyErr_Occurred` will return false. .. c:function:: PyObject* PyObject_GetIter(PyObject *o) This is equivalent to the Python expression ``iter(o)``. It returns a new iterator for the object argument, or the object itself if the object is already - an iterator. Raises :exc:`TypeError` and returns *NULL* if the object cannot be + an iterator. Raises :exc:`TypeError` and returns ``NULL`` if the object cannot be iterated. diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 6b07c87d62eded..0df12c67f40bc3 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -13,22 +13,22 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XINCREF`. + Increment the reference count for object *o*. The object must not be ``NULL``; if + you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be *NULL*, in + Increment the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XDECREF`. If the reference + Decrement the reference count for object *o*. The object must not be ``NULL``; if + you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. If the reference count reaches zero, the object's type's deallocation function (which must not be - *NULL*) is invoked. + ``NULL``) is invoked. .. warning:: @@ -44,18 +44,18 @@ objects. .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be *NULL*, in + Decrement the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, and the same warning applies. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be *NULL*, in + Decrement the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, except that the argument is also set to *NULL*. The warning + :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because - the macro carefully uses a temporary variable and sets the argument to *NULL* + the macro carefully uses a temporary variable and sets the argument to ``NULL`` before decrementing its reference count. It is a good idea to use this macro whenever decrementing the reference diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 080ea3222cfbb2..1d86de66ed3eda 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -14,18 +14,18 @@ Reflection .. c:function:: PyObject* PyEval_GetLocals() Return a dictionary of the local variables in the current execution frame, - or *NULL* if no frame is currently executing. + or ``NULL`` if no frame is currently executing. .. c:function:: PyObject* PyEval_GetGlobals() Return a dictionary of the global variables in the current execution frame, - or *NULL* if no frame is currently executing. + or ``NULL`` if no frame is currently executing. .. c:function:: PyFrameObject* PyEval_GetFrame() - Return the current thread state's frame, which is *NULL* if no frame is + Return the current thread state's frame, which is ``NULL`` if no frame is currently executing. diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index d11a2dde54dd9e..65818859041179 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -26,39 +26,39 @@ Sequence Protocol .. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. This is the equivalent of the Python expression ``o1 + o2``. .. c:function:: PyObject* PySequence_Repeat(PyObject *o, Py_ssize_t count) - Return the result of repeating sequence object *o* *count* times, or *NULL* on + Return the result of repeating sequence object *o* *count* times, or ``NULL`` on failure. This is the equivalent of the Python expression ``o * count``. .. c:function:: PyObject* PySequence_InPlaceConcat(PyObject *o1, PyObject *o2) - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. + Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python expression ``o1 += o2``. .. c:function:: PyObject* PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) - Return the result of repeating sequence object *o* *count* times, or *NULL* on + Return the result of repeating sequence object *o* *count* times, or ``NULL`` on failure. The operation is done *in-place* when *o* supports it. This is the equivalent of the Python expression ``o *= count``. .. c:function:: PyObject* PySequence_GetItem(PyObject *o, Py_ssize_t i) - Return the *i*\ th element of *o*, or *NULL* on failure. This is the equivalent of + Return the *i*\ th element of *o*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o[i]``. .. c:function:: PyObject* PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - Return the slice of sequence object *o* between *i1* and *i2*, or *NULL* on + Return the slice of sequence object *o* between *i1* and *i2*, or ``NULL`` on failure. This is the equivalent of the Python expression ``o[i1:i2]``. @@ -69,7 +69,7 @@ Sequence Protocol is the equivalent of the Python statement ``o[i] = v``. This function *does not* steal a reference to *v*. - If *v* is *NULL*, the element is deleted, however this feature is + If *v* is ``NULL``, the element is deleted, however this feature is deprecated in favour of using :c:func:`PySequence_DelItem`. @@ -114,7 +114,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_List(PyObject *o) Return a list object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. The returned list is guaranteed to be new. This is + or ``NULL`` on failure. The returned list is guaranteed to be new. This is equivalent to the Python expression ``list(o)``. @@ -123,38 +123,45 @@ Sequence Protocol .. index:: builtin: tuple Return a tuple object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. If *o* is a tuple, a new reference will be returned, + or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, otherwise a tuple will be constructed with the appropriate contents. This is equivalent to the Python expression ``tuple(o)``. .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in - which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access - the members of the result. Returns *NULL* on failure. If the object is not - a sequence or iterable, raises :exc:`TypeError` with *m* as the message text. + Return the sequence or iterable *o* as an object usable by the other + ``PySequence_Fast*`` family of functions. If the object is not a sequence or + iterable, raises :exc:`TypeError` with *m* as the message text. Returns + ``NULL`` on failure. + + The ``PySequence_Fast*`` functions are thus named because they assume + *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access + the data fields of *o* directly. + + As a CPython implementation detail, if *o* is already a sequence or list, it + will be returned. .. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o) Returns the length of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be + :c:func:`PySequence_Fast` and that *o* is not ``NULL``. The size can also be gotten by calling :c:func:`PySequence_Size` on *o*, but - :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list - or tuple. + :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a + list or tuple. .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) Return the *i*\ th element of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast`, *o* is not *NULL*, and that *i* is within bounds. + :c:func:`PySequence_Fast`, *o* is not ``NULL``, and that *i* is within bounds. .. c:function:: PyObject** PySequence_Fast_ITEMS(PyObject *o) Return the underlying array of PyObject pointers. Assumes that *o* was returned - by :c:func:`PySequence_Fast` and *o* is not *NULL*. + by :c:func:`PySequence_Fast` and *o* is not ``NULL``. Note, if a list gets resized, the reallocation may relocate the items array. So, only use the underlying array pointer in contexts where the sequence @@ -163,7 +170,7 @@ Sequence Protocol .. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i) - Return the *i*\ th element of *o* or *NULL* on failure. Macro form of + Return the *i*\ th element of *o* or ``NULL`` on failure. Faster form of :c:func:`PySequence_GetItem` but without checking that :c:func:`PySequence_Check` on *o* is true and without adjustment for negative indices. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 074fcb877e1da7..54819e8fd6cbdc 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -80,8 +80,8 @@ the constructor functions work with any iterable Python object. .. c:function:: PyObject* PySet_New(PyObject *iterable) Return a new :class:`set` containing objects returned by the *iterable*. The - *iterable* may be *NULL* to create a new empty set. Return the new set on - success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is not + *iterable* may be ``NULL`` to create a new empty set. Return the new set on + success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. The constructor is also useful for copying a set (``c=set(s)``). @@ -89,8 +89,8 @@ the constructor functions work with any iterable Python object. .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) Return a new :class:`frozenset` containing objects returned by the *iterable*. - The *iterable* may be *NULL* to create a new empty frozenset. Return the new - set on success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is + The *iterable* may be ``NULL`` to create a new empty frozenset. Return the new + set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. @@ -149,7 +149,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: PyObject* PySet_Pop(PyObject *set) Return a new reference to an arbitrary object in the *set*, and removes the - object from the *set*. Return *NULL* on failure. Raise :exc:`KeyError` if the + object from the *set*. Return ``NULL`` on failure. Raise :exc:`KeyError` if the set is empty. Raise a :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index d924308890b8d0..48a58c6c6f7e36 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -14,15 +14,15 @@ Slice Objects .. c:function:: int PySlice_Check(PyObject *ob) - Return true if *ob* is a slice object; *ob* must not be *NULL*. + Return true if *ob* is a slice object; *ob* must not be ``NULL``. .. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of - the same names. Any of the values may be *NULL*, in which case the - ``None`` will be used for the corresponding attribute. Return *NULL* if + the same names. Any of the values may be ``NULL``, in which case the + ``None`` will be used for the corresponding attribute. Return ``NULL`` if the new object could not be allocated. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 5e0cfd0264f931..e9c276ce782900 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -105,19 +105,29 @@ the definition of all other Python objects. .. c:type:: PyCFunction Type of the functions used to implement most Python callables in C. - Functions of this type take two :c:type:`PyObject\*` parameters and return - one such value. If the return value is *NULL*, an exception shall have - been set. If not *NULL*, the return value is interpreted as the return + Functions of this type take two :c:type:`PyObject*` parameters and return + one such value. If the return value is ``NULL``, an exception shall have + been set. If not ``NULL``, the return value is interpreted as the return value of the function as exposed in Python. The function must return a new reference. .. c:type:: PyCFunctionWithKeywords - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. + Type of the functions used to implement Python callables in C + with signature :const:`METH_VARARGS | METH_KEYWORDS`. + + +.. c:type:: _PyCFunctionFast + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL`. + + +.. c:type:: _PyCFunctionFastWithKeywords + + Type of the functions used to implement Python callables in C + with signature :const:`METH_FASTCALL | METH_KEYWORDS`. .. c:type:: PyMethodDef @@ -141,36 +151,65 @@ the definition of all other Python objects. +------------------+---------------+-------------------------------+ The :attr:`ml_meth` is a C function pointer. The functions may be of different -types, but they always return :c:type:`PyObject\*`. If the function is not of +types, but they always return :c:type:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as -:c:type:`PyObject\*`, it is common that the method implementation uses the +:c:type:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. +convention. +There are four basic calling conventions for positional arguments +and two of them can be combined with :const:`METH_KEYWORDS` to support +also keyword arguments. So there are a total of 6 calling conventions: .. data:: METH_VARARGS This is the typical calling convention, where the methods have the type - :c:type:`PyCFunction`. The function expects two :c:type:`PyObject\*` values. + :c:type:`PyCFunction`. The function expects two :c:type:`PyObject*` values. The first one is the *self* object for methods; for module functions, it is the module object. The second parameter (often called *args*) is a tuple object representing all arguments. This parameter is typically processed using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. -.. data:: METH_KEYWORDS +.. data:: METH_VARARGS | METH_KEYWORDS Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. + The function expects three parameters: *self*, *args*, *kwargs* where + *kwargs* is a dictionary of all the keyword arguments or possibly ``NULL`` + if there are no keyword arguments. The parameters are typically processed + using :c:func:`PyArg_ParseTupleAndKeywords`. + + +.. data:: METH_FASTCALL + + Fast calling convention supporting only positional arguments. + The methods have the type :c:type:`_PyCFunctionFast`. + The first parameter is *self*, the second parameter is a C array + of :c:type:`PyObject*` values indicating the arguments and the third + parameter is the number of arguments (the length of the array). + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 + + +.. data:: METH_FASTCALL | METH_KEYWORDS + + Extension of :const:`METH_FASTCALL` supporting also keyword arguments, + with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + Keyword arguments are passed the same way as in the vectorcall protocol: + there is an additional fourth :c:type:`PyObject*` parameter + which is a tuple representing the names of the keyword arguments + or possibly ``NULL`` if there are no keywords. The values of the keyword + arguments are stored in the *args* array, after the positional arguments. + + This is not part of the :ref:`limited API `. + + .. versionadded:: 3.7 .. data:: METH_NOARGS @@ -179,7 +218,7 @@ can be combined with a binding flag. they are listed with the :const:`METH_NOARGS` flag. They need to be of type :c:type:`PyCFunction`. The first parameter is typically named *self* and will hold a reference to the module or object instance. In all cases the second - parameter will be *NULL*. + parameter will be ``NULL``. .. data:: METH_O @@ -187,7 +226,7 @@ can be combined with a binding flag. Methods with a single object argument can be listed with the :const:`METH_O` flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. They have the type :c:type:`PyCFunction`, with the *self* parameter, and a - :c:type:`PyObject\*` parameter representing the single argument. + :c:type:`PyObject*` parameter representing the single argument. These two constants are not used to indicate the calling convention but the @@ -210,7 +249,7 @@ method. .. index:: builtin: staticmethod - The method will be passed *NULL* as the first parameter rather than an + The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to what is created when using the :func:`staticmethod` built-in function. @@ -284,7 +323,7 @@ definition with the same method name. =============== ================== :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that - :c:macro:`T_OBJECT` returns ``None`` if the member is *NULL* and + :c:macro:`T_OBJECT` returns ``None`` if the member is ``NULL`` and :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX` handles use of the :keyword:`del` statement on that attribute more correctly @@ -294,7 +333,7 @@ definition with the same method name. read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` - members can be deleted. (They are set to *NULL*). + members can be deleted. (They are set to ``NULL``). .. c:type:: PyGetSetDef @@ -320,18 +359,18 @@ definition with the same method name. | | | getter and setter | +-------------+------------------+-----------------------------------+ - The ``get`` function takes one :c:type:`PyObject\*` parameter (the + The ``get`` function takes one :c:type:`PyObject*` parameter (the instance) and a function pointer (the associated ``closure``):: typedef PyObject *(*getter)(PyObject *, void *); - It should return a new reference on success or *NULL* with a set exception + It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and + ``set`` functions take two :c:type:`PyObject*` parameters (the instance and the value to be set) and a function pointer (the associated ``closure``):: typedef int (*setter)(PyObject *, PyObject *, void *); - In case the attribute should be deleted the second parameter is *NULL*. + In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 7d870a8d4e4a71..c851ff66487d5c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -22,7 +22,7 @@ Operating System Utilities Return true (nonzero) if the standard I/O file *fp* with name *filename* is deemed interactive. This is the case for files for which ``isatty(fileno(fp))`` is true. If the global flag :c:data:`Py_InteractiveFlag` is true, this function - also returns true if the *filename* pointer is *NULL* or if the name is equal to + also returns true if the *filename* pointer is ``NULL`` or if the name is equal to one of the strings ``''`` or ``'???'``. @@ -33,6 +33,12 @@ Operating System Utilities that clones the current process. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_BeforeFork()``. + .. versionadded:: 3.7 @@ -44,6 +50,12 @@ Operating System Utilities of whether process cloning was successful. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_AfterFork_Parent()``. + .. versionadded:: 3.7 @@ -55,6 +67,12 @@ Operating System Utilities any chance the process will call back into the Python interpreter. Only available on systems where :c:func:`fork` is defined. + .. warning:: + The C :c:func:`fork` call should only be made from the + :ref:`"main" thread ` (of the + :ref:`"main" interpreter `). The same is + true for ``PyOS_AfterFork_Child()``. + .. versionadded:: 3.7 .. seealso:: @@ -201,12 +219,12 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or *NULL* if it does + Return the object *name* from the :mod:`sys` module or ``NULL`` if it does not exist, without setting an exception. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) - Set *name* in the :mod:`sys` module to *v* unless *v* is *NULL*, in which + Set *name* in the :mod:`sys` module to *v* unless *v* is ``NULL``, in which case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. @@ -283,7 +301,7 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: PyObject *PySys_GetXOptions() Return the current dictionary of :option:`-X` options, similarly to - :data:`sys._xoptions`. On error, *NULL* is returned and an exception is + :data:`sys._xoptions`. On error, ``NULL`` is returned and an exception is set. .. versionadded:: 3.2 @@ -291,9 +309,7 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: int PySys_Audit(const char *event, const char *format, ...) - .. index:: single: audit events - - Raises an auditing event with any active hooks. Returns zero for success + Raise an auditing event with any active hooks. Return zero for success and non-zero with an exception set on failure. If any hooks have been added, *format* and other arguments will be used @@ -304,20 +320,31 @@ accessible to C code. They all work with the current interpreter thread's arguments to this function will be consumed, using it may cause reference leaks.) + Note that ``#`` format characters should always be treated as + ``Py_ssize_t``, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. + :func:`sys.audit` performs the same function from Python code. .. versionadded:: 3.8 + .. versionchanged:: 3.8.2 -.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) + Require ``Py_ssize_t`` for ``#`` format characters. Previously, an + unavoidable deprecation warning was raised. - .. index:: single: audit events - Adds to the collection of active auditing hooks. Returns zero for success - and non-zero on failure. If the runtime has been initialized, also sets an +.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) + + Append the callable *hook* to the list of active auditing hooks. + Return zero for success + and non-zero on failure. If the runtime has been initialized, also set an error on failure. Hooks added through this API are called for all interpreters created by the runtime. + The *userData* pointer is passed into the hook function. Since hook + functions may be called from different runtimes, this pointer should not + refer directly to Python state. + This function is safe to call before :c:func:`Py_Initialize`. When called after runtime initialization, existing audit hooks are notified and may silently abort the operation by raising an error subclassed from @@ -328,13 +355,18 @@ accessible to C code. They all work with the current interpreter thread's :c:type:`PyTupleObject`. The hook function is always called with the GIL held by the Python interpreter that raised the event. - The *userData* pointer is passed into the hook function. Since hook - functions may be called from different runtimes, this pointer should not - refer directly to Python state. + See :pep:`578` for a detailed description of auditing. Functions in the + runtime and standard library that raise events are listed in the + :ref:`audit events table `. + Details are in each function's documentation. + + .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - See :pep:`578` for a detailed description of auditing. Functions in the - runtime and standard library that raise events include the details in each - function's documentation. + If the interpreter is initialized, this function raises a auditing event + ``sys.addaudithook`` with no arguments. If any existing hooks raise an + exception derived from :class:`Exception`, the new hook will not be + added and the exception is cleared. As a result, callers cannot assume + that their hook has been added unless they control all existing hooks. .. versionadded:: 3.8 diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 259ec4fb48515b..d7acc4eaa979af 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -33,12 +33,12 @@ Tuple Objects .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - Return a new tuple object of size *len*, or *NULL* on failure. + Return a new tuple object of size *len*, or ``NULL`` on failure. .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - Return a new tuple object of size *n*, or *NULL* on failure. The tuple values + Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values are initialized to the subsequent *n* C arguments pointing to Python objects. ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. @@ -50,14 +50,14 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - Return the size of the tuple *p*, which must be non-*NULL* and point to a tuple; + Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; no error checking is performed. .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is - out of bounds, return *NULL* and sets an :exc:`IndexError` exception. + out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -67,18 +67,21 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) - Take a slice of the tuple pointed to by *p* from *low* to *high* and return it - as a new tuple. + Return the slice of the tuple pointed to by *p* between *low* and *high*, + or ``NULL`` on failure. This is the equivalent of the Python expression + ``p[low:high]``. Indexing from the end of the list is not supported. .. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) Insert a reference to object *o* at position *pos* of the tuple pointed to by - *p*. Return ``0`` on success. + *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` + and set an :exc:`IndexError` exception. .. note:: - This function "steals" a reference to *o*. + This function "steals" a reference to *o* and discards a reference to + an item already in the tuple at the affected position. .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -88,7 +91,10 @@ Tuple Objects .. note:: - This function "steals" a reference to *o*. + This macro "steals" a reference to *o*, and, unlike + :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that + is being replaced; any reference in the tuple at position *pos* will be + leaked. .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) @@ -101,7 +107,7 @@ Tuple Objects only more efficiently. Returns ``0`` on success. Client code should never assume that the resulting value of ``*p`` will be the same as before calling this function. If the object referenced by ``*p`` is replaced, the original - ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to *NULL*, and + ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to ``NULL``, and raises :exc:`MemoryError` or :exc:`SystemError`. @@ -141,39 +147,39 @@ type. Contains the meta information of a struct sequence type to create. - +-------------------+------------------------------+------------------------------------+ - | Field | C Type | Meaning | - +===================+==============================+====================================+ - | ``name`` | ``const char *`` | name of the struct sequence type | - +-------------------+------------------------------+------------------------------------+ - | ``doc`` | ``const char *`` | pointer to docstring for the type | - | | | or NULL to omit | - +-------------------+------------------------------+------------------------------------+ - | ``fields`` | ``PyStructSequence_Field *`` | pointer to *NULL*-terminated array | - | | | with field names of the new type | - +-------------------+------------------------------+------------------------------------+ - | ``n_in_sequence`` | ``int`` | number of fields visible to the | - | | | Python side (if used as tuple) | - +-------------------+------------------------------+------------------------------------+ + +-------------------+------------------------------+--------------------------------------+ + | Field | C Type | Meaning | + +===================+==============================+======================================+ + | ``name`` | ``const char *`` | name of the struct sequence type | + +-------------------+------------------------------+--------------------------------------+ + | ``doc`` | ``const char *`` | pointer to docstring for the type | + | | | or ``NULL`` to omit | + +-------------------+------------------------------+--------------------------------------+ + | ``fields`` | ``PyStructSequence_Field *`` | pointer to ``NULL``-terminated array | + | | | with field names of the new type | + +-------------------+------------------------------+--------------------------------------+ + | ``n_in_sequence`` | ``int`` | number of fields visible to the | + | | | Python side (if used as tuple) | + +-------------------+------------------------------+--------------------------------------+ .. c:type:: PyStructSequence_Field Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:type:`PyObject\*`. The index in the + tuple, all fields are typed as :c:type:`PyObject*`. The index in the :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which field of the struct sequence is described. - +-----------+------------------+--------------------------------------+ - | Field | C Type | Meaning | - +===========+==================+======================================+ - | ``name`` | ``const char *`` | name for the field or *NULL* to end | - | | | the list of named fields, set to | - | | | PyStructSequence_UnnamedField to | - | | | leave unnamed | - +-----------+------------------+--------------------------------------+ - | ``doc`` | ``const char *`` | field docstring or *NULL* to omit | - +-----------+------------------+--------------------------------------+ + +-----------+------------------+-----------------------------------------+ + | Field | C Type | Meaning | + +===========+==================+=========================================+ + | ``name`` | ``const char *`` | name for the field or ``NULL`` to end | + | | | the list of named fields, set to | + | | | :c:data:`PyStructSequence_UnnamedField` | + | | | to leave unnamed | + +-----------+------------------+-----------------------------------------+ + | ``doc`` | ``const char *`` | field docstring or ``NULL`` to omit | + +-----------+------------------+-----------------------------------------+ .. c:var:: char* PyStructSequence_UnnamedField diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 8f8367ab77c8c4..2a6d618858a970 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -81,7 +81,7 @@ Type Objects Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to *NULL*. + initialize all its contents to ``NULL``. .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -98,7 +98,7 @@ Type Objects .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the - result is *NULL*, this indicates that either the slot is *NULL*, + result is ``NULL``, this indicates that either the slot is ``NULL``, or that the function was called with invalid parameters. Callers will typically cast the result pointer into the appropriate function type. @@ -118,13 +118,15 @@ The following functions and structs are used to create .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - Creates and returns a heap type object from the *spec*. + Creates and returns a heap type object from the *spec* + (:const:`Py_TPFLAGS_HEAPTYPE`). If *bases* is a tuple, the created heap type contains all types contained in it as base types. - If *bases* is *NULL*, the *Py_tp_base* slot is used instead. - If that also is *NULL*, the new type derives from :class:`object`. + If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. + If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If that also is ``NULL``, the new type derives from :class:`object`. This function calls :c:func:`PyType_Ready` on the new type. @@ -142,10 +144,6 @@ The following functions and structs are used to create Name of the type, used to set :c:member:`PyTypeObject.tp_name`. - .. c:member:: const char* PyType_Spec.doc - - Type docstring, used to set :c:member:`PyTypeObject.tp_doc`. - .. c:member:: int PyType_Spec.basicsize .. c:member:: int PyType_Spec.itemsize @@ -184,7 +182,7 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - The following fields cannot be set using *PyType_Spec* and *PyType_Slot*: + The following fields cannot be set using :c:type:`PyType_Spec` and :c:type:`PyType_Slot`: * :c:member:`~PyTypeObject.tp_dict` * :c:member:`~PyTypeObject.tp_mro` @@ -197,7 +195,8 @@ The following functions and structs are used to create * :c:member:`~PyBufferProcs.bf_getbuffer` * :c:member:`~PyBufferProcs.bf_releasebuffer` - Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be + problematic on some platforms. To avoid issues, use the *bases* argument of :py:func:`PyType_FromSpecWithBases` instead. @@ -206,4 +205,4 @@ The following functions and structs are used to create The desired value of the slot. In most cases, this is a pointer to a function. - May not be *NULL*. + May not be ``NULL``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 83fcc5abed7079..3d186231b26b70 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -20,7 +20,7 @@ functionality. The fields of the type object are examined in detail in this section. The fields will be described in the order in which they occur in the structure. -In addition to the following quick reference, the :ref:`examples` +In addition to the following quick reference, the :ref:`typedef-examples` section provides at-a-glance insight into the meaning and use of :c:type:`PyTypeObject`. @@ -159,20 +159,20 @@ fields exist as well: A slot name in parentheses indicates it is (effectively) deprecated. Names in angle brackets should be treated as read-only. Names in square brackets are for internal use only. - "" (as a prefix) means the field is required (must be non-*NULL*). + "" (as a prefix) means the field is required (must be non-``NULL``). .. [#cols] Columns: **"O"**: set on :c:type:`PyBaseObject_Type` **"T"**: set on :c:type:`PyType_Type` - **"D"**: default (if slot is set to *NULL*) + **"D"**: default (if slot is set to ``NULL``) .. code-block:: none - X - *PyType_Ready* sets this value if it is *NULL* - ~ - *PyType_Ready* always sets this value (it should be *NULL*) - ? - *PyType_Ready* may set this value depending on other slots + X - PyType_Ready sets this value if it is NULL + ~ - PyType_Ready always sets this value (it should be NULL) + ? - PyType_Ready may set this value depending on other slots Also see the inheritance column ("I"). @@ -180,7 +180,7 @@ fields exist as well: .. code-block:: none - X - type slot is inherited via *PyType_Ready* if defined with a *NULL* value + X - type slot is inherited via PyType_Ready if defined with a NULL value % - the slots of the sub-struct are inherited individually G - inherited, but only in combination with other slots; see the slot's description ? - it's complicated; see the slot's description @@ -285,7 +285,7 @@ sub-slots +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __truediv__ | +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`binaryfunc` | __index__ | + | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | | | | __rmatmul__ | @@ -491,8 +491,8 @@ type objects) *must* have the :attr:`ob_size` field. PyObject* PyObject._ob_prev These fields are only present when the macro ``Py_TRACE_REFS`` is defined. - Their initialization to *NULL* is taken care of by the ``PyObject_HEAD_INIT`` - macro. For statically allocated objects, these fields always remain *NULL*. + Their initialization to ``NULL`` is taken care of by the ``PyObject_HEAD_INIT`` + macro. For statically allocated objects, these fields always remain ``NULL``. For dynamically allocated objects, these two fields are used to link the object into a doubly-linked list of *all* live objects on the heap. This could be used for various debugging purposes; currently the only use is to print the objects @@ -523,7 +523,7 @@ type objects) *must* have the :attr:`ob_size` field. argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be ``&PyType_Type``. However, for dynamically loadable extension modules that must be usable on Windows (at least), the compiler complains that this is not a valid - initializer. Therefore, the convention is to pass *NULL* to the + initializer. Therefore, the convention is to pass ``NULL`` to the ``PyObject_HEAD_INIT`` macro and to initialize this field explicitly at the start of the module's initialization function, before doing anything else. This is typically done like this:: @@ -531,7 +531,7 @@ type objects) *must* have the :attr:`ob_size` field. Foo_Type.ob_type = &PyType_Type; This should be done before any instances of the type are created. - :c:func:`PyType_Ready` checks if :attr:`ob_type` is *NULL*, and if so, + :c:func:`PyType_Ready` checks if :attr:`ob_type` is ``NULL``, and if so, initializes it to the :attr:`ob_type` field of the base class. :c:func:`PyType_Ready` will not change this field if it is non-zero. @@ -557,7 +557,7 @@ PyTypeObject Slots ------------------ Each slot has a section describing inheritance. If :c:func:`PyType_Ready` -may set a value when the field is set to *NULL* then there will also be +may set a value when the field is set to ``NULL`` then there will also be a "Default" section. (Note that many fields set on :c:type:`PyBaseObject_Type` and :c:type:`PyType_Type` effectively act as defaults.) @@ -586,7 +586,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. - This field must not be *NULL*. It is the only required field + This field must not be ``NULL``. It is the only required field in :c:func:`PyTypeObject` (other than potentially :c:member:`~PyTypeObject.tp_itemsize`). @@ -631,7 +631,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the alignment requirement for ``double``). - For any type with variable-length instances, this field must not be *NULL*. + For any type with variable-length instances, this field must not be ``NULL``. **Inheritance:** @@ -654,9 +654,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) the instance is still in existence, but there are no references to it. The destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not - subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is + to the allocation function used to allocate the buffer), and call the type's + :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable + (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated @@ -664,6 +664,21 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`PyObject_GC_Del` if the instance was allocated using :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. + Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should decrement the reference count for its type object after + calling the type deallocator. In order to avoid dangling pointers, the + recommended way to achieve this is: + + .. code-block:: c + + static void foo_dealloc(foo_object *self) { + PyTypeObject *tp = Py_TYPE(self); + // free references and buffers here + tp->tp_free(self); + Py_DECREF(tp); + } + + **Inheritance:** This field is inherited by subtypes. @@ -725,7 +740,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_call`: a subtype inherits :c:member:`~PyTypeObject.tp_vectorcall_offset` from its base type when - the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL. + the subtype’s :c:member:`~PyTypeObject.tp_call` is ``NULL``. Note that `heap types`_ (including subclasses defined in Python) do not inherit the :const:`_Py_TPFLAGS_HAVE_VECTORCALL` flag. @@ -745,7 +760,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. .. c:member:: setattrfunc PyTypeObject.tp_setattr @@ -762,7 +777,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. .. c:member:: PyAsyncMethods* PyTypeObject.tp_as_async @@ -879,13 +894,13 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both *NULL*. + :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. .. c:member:: ternaryfunc PyTypeObject.tp_call An optional pointer to a function that implements calling the object. This - should be *NULL* if the object is not callable. The signature is the same as + should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: PyObject *tp_call(PyObject *self, PyObject *args, PyObject *kwargs); @@ -937,7 +952,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. **Default:** @@ -952,7 +967,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) PyObject *tp_setattro(PyObject *self, PyObject *attr, PyObject *value); - In addition, setting *value* to *NULL* to delete an attribute must be + In addition, setting *value* to ``NULL`` to delete an attribute must be supported. It is usually convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which implements the normal way of setting object attributes. @@ -963,7 +978,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. **Default:** @@ -990,7 +1005,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if such a flag bit is clear, the type fields it guards must not be accessed and - must be considered to have a zero or *NULL* value instead. + must be considered to have a zero or ``NULL`` value instead. **Inheritance:** @@ -1003,7 +1018,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have - *NULL* values. + ``NULL`` values. .. XXX are most flag bits *really* inherited individually? @@ -1021,7 +1036,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. data:: Py_TPFLAGS_HEAPTYPE - This bit is set when the type object itself is allocated on the heap. In this + This bit is set when the type object itself is allocated on the heap, for + example, types created dynamically using :c:func:`PyType_FromSpec`. In this case, the :attr:`ob_type` field of its instances is considered a reference to the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of @@ -1081,7 +1097,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) together with the :attr:`tp_traverse` and :attr:`tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :attr:`tp_traverse` and - :attr:`tp_clear` fields in the subtype exist and have *NULL* + :attr:`tp_clear` fields in the subtype exist and have ``NULL`` values. @@ -1164,7 +1180,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This bit is set on *static* subtypes if ``tp_flags`` is not overridden: a subtype inherits ``_Py_TPFLAGS_HAVE_VECTORCALL`` from its base type - when the subtype’s :c:member:`~PyTypeObject.tp_call` is NULL + when the subtype’s :c:member:`~PyTypeObject.tp_call` is ``NULL`` and the subtype's ``Py_TPFLAGS_HEAPTYPE`` is not set. `Heap types`_ do not inherit ``_Py_TPFLAGS_HAVE_VECTORCALL``. @@ -1202,7 +1218,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects. For example, this is function :c:func:`local_traverse` from the + objects that the instance owns. For example, this is function :c:func:`local_traverse` from the :mod:`_thread` extension module:: static int @@ -1216,12 +1232,24 @@ and :c:type:`PyType_Type` effectively act as defaults.) Note that :c:func:`Py_VISIT` is called only on those members that can participate in reference cycles. Although there is also a ``self->key`` member, it can only - be *NULL* or a Python string and therefore cannot be part of a reference cycle. + be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. On the other hand, even if you know a member can never be part of a cycle, as a debugging aid you may want to visit it anyway just so the :mod:`gc` module's :func:`~gc.get_referents` function will include it. + .. warning:: + When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members + that the instance *owns* (by having strong references to them) must be + visited. For instance, if an object supports weak references via the + :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting + the linked list (what *tp_weaklist* points to) must **not** be + visited as the instance does not directly own the weak references to itself + (the weakreference list is there to support the weak reference machinery, + but the instance has no strong reference to the elements inside it, as they + are allowed to be removed even if the instance is still alive). + + Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just anything. @@ -1255,7 +1283,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those - members to *NULL*, as in the following example:: + members to ``NULL``, as in the following example:: static int local_clear(localobject *self) @@ -1269,12 +1297,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) The :c:func:`Py_CLEAR` macro should be used, because clearing references is delicate: the reference to the contained object must not be decremented until - after the pointer to the contained object is set to *NULL*. This is because + after the pointer to the contained object is set to ``NULL``. This is because decrementing the reference count may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, - it's important that the pointer to the contained object be *NULL* at that time, + it's important that the pointer to the contained object be ``NULL`` at that time, so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. @@ -1308,15 +1336,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) The function should return the result of the comparison (usually ``Py_True`` or ``Py_False``). If the comparison is undefined, it must return - ``Py_NotImplemented``, if another error occurred it must return *NULL* and + ``Py_NotImplemented``, if another error occurred it must return ``NULL`` and set an exception condition. - .. note:: - - If you want to implement a type for which only a limited set of - comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and - friends), directly raise :exc:`TypeError` in the rich comparison function. - The following constants are defined to be used as the third argument for :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: @@ -1338,7 +1360,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The following macro is defined to ease writing rich comparison functions: - .. c:function:: PyObject \*Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, int op) + .. c:macro:: Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, op) Return ``Py_True`` or ``Py_False`` from the function, depending on the result of a comparison. @@ -1348,7 +1370,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The return value's reference count is properly incremented. - On error, sets an exception and returns *NULL* from the function. + On error, sets an exception and returns ``NULL`` from the function. .. versionadded:: 3.7 @@ -1359,7 +1381,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both - *NULL*. + ``NULL``. **Default:** @@ -1376,8 +1398,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) than zero and contains the offset in the instance structure of the weak reference list head (ignoring the GC header, if present); this offset is used by :c:func:`PyObject_ClearWeakRefs` and the :c:func:`PyWeakref_\*` functions. The - instance structure needs to include a field of type :c:type:`PyObject\*` which is - initialized to *NULL*. + instance structure needs to include a field of type :c:type:`PyObject*` which is + initialized to ``NULL``. Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for weak references to the type object itself. @@ -1426,9 +1448,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) PyObject *tp_iternext(PyObject *self); - When the iterator is exhausted, it must return *NULL*; a :exc:`StopIteration` + When the iterator is exhausted, it must return ``NULL``; a :exc:`StopIteration` exception may or may not be set. When another error occurs, it must return - *NULL* too. Its presence signals that the instances of this type are + ``NULL`` too. Its presence signals that the instances of this type are iterators. Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that @@ -1444,7 +1466,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMethodDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. For each entry in the array, an entry is added to the type's dictionary (see @@ -1458,7 +1480,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMemberDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1473,7 +1495,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset - An optional pointer to a static *NULL*-terminated array of :c:type:`PyGetSetDef` + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. For each entry in the array, an entry is added to the type's dictionary (see @@ -1523,7 +1545,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The type's dictionary is stored here by :c:func:`PyType_Ready`. - This field should normally be initialized to *NULL* before PyType_Ready is + This field should normally be initialized to ``NULL`` before PyType_Ready is called; it may also be initialized to a dictionary containing initial attributes for the type. Once :c:func:`PyType_Ready` has initialized the type, extra attributes for the type may be added to this dictionary only if they don't @@ -1536,7 +1558,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - If this field is *NULL*, :c:func:`PyType_Ready` will assign a new + If this field is ``NULL``, :c:func:`PyType_Ready` will assign a new dictionary to it. .. warning:: @@ -1569,7 +1591,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) int tp_descr_set(PyObject *self, PyObject *obj, PyObject *value); - The *value* argument is set to *NULL* to delete the value. + The *value* argument is set to ``NULL`` to delete the value. .. XXX explain more? @@ -1635,7 +1657,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** This slot has no default. For static types, if the field is - *NULL* then no :attr:`__dict__` gets created for instances. + ``NULL`` then no :attr:`__dict__` gets created for instances. .. c:member:: initproc PyTypeObject.tp_init @@ -1655,7 +1677,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) arguments represent positional and keyword arguments of the call to :meth:`__init__`. - The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is + The :c:member:`~PyTypeObject.tp_init` function, if not ``NULL``, is called when an instance is created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an instance of some other type that is not a subtype of the original type, no @@ -1722,12 +1744,12 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** This field is inherited by subtypes, except it is not inherited by static types - whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. + whose :c:member:`~PyTypeObject.tp_base` is ``NULL`` or ``&PyBaseObject_Type``. **Default:** For static types this field has no default. This means if the - slot is defined as *NULL*, the type cannot be called to create new + slot is defined as ``NULL``, the type cannot be called to create new instances; presumably there is some other way to create instances, like a factory function. @@ -1778,7 +1800,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - This slot has no default. If this field is *NULL*, + This slot has no default. If this field is ``NULL``, :const:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent. @@ -1786,7 +1808,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple of base types. - This is set for types created by a class statement. It should be *NULL* for + This is set for types created by a class statement. It should be ``NULL`` for statically defined types. **Inheritance:** @@ -1945,7 +1967,7 @@ This results in types that are limited relative to types defined in Python: :ref:`sub-interpreters `, so they should not include any subinterpreter-specific state. -Also, since *PyTypeObject* is not part of the :ref:`stable ABI `, +Also, since :c:type:`PyTypeObject` is not part of the :ref:`stable ABI `, any extension modules using static types must be compiled for a specific Python minor version. @@ -2024,12 +2046,12 @@ Number Object Structures and implement the necessary conversions (at least one of the operands is an instance of the defined type). If the operation is not defined for the given operands, binary and ternary functions must return - ``Py_NotImplemented``, if another error occurred they must return *NULL* + ``Py_NotImplemented``, if another error occurred they must return ``NULL`` and set an exception. .. note:: - The :c:data:`nb_reserved` field should always be *NULL*. It + The :c:data:`nb_reserved` field should always be ``NULL``. It was previously called :c:data:`nb_long`, and was renamed in Python 3.0.1. @@ -2088,14 +2110,14 @@ Mapping Object Structures This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to - *NULL* if the object has no defined length. + ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the - :c:func:`PyMapping_Check` function to return ``1``, it can be *NULL* + :c:func:`PyMapping_Check` function to return ``1``, it can be ``NULL`` otherwise. .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript @@ -2103,8 +2125,8 @@ Mapping Object Structures This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and :c:func:`PyObject_DelSlice`. It has the same signature as - :c:func:`!PyObject_SetItem`, but *v* can also be set to *NULL* to delete - an item. If this slot is *NULL*, the object does not support item + :c:func:`!PyObject_SetItem`, but *v* can also be set to ``NULL`` to delete + an item. If this slot is ``NULL``, the object does not support item assignment and deletion. @@ -2146,11 +2168,11 @@ Sequence Object Structures signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. This slot must be filled for the :c:func:`PySequence_Check` - function to return ``1``, it can be *NULL* otherwise. + function to return ``1``, it can be ``NULL`` otherwise. Negative indexes are handled as follows: if the :attr:`sq_length` slot is filled, it is called and the sequence length is used to compute a positive - index which is passed to :attr:`sq_item`. If :attr:`sq_length` is *NULL*, + index which is passed to :attr:`sq_item`. If :attr:`sq_length` is ``NULL``, the index is passed as is to the function. .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item @@ -2159,13 +2181,13 @@ Sequence Object Structures signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot. - This slot may be left to *NULL* if the object does not support + This slot may be left to ``NULL`` if the object does not support item assignment and deletion. .. c:member:: objobjproc PySequenceMethods.sq_contains This function may be used by :c:func:`PySequence_Contains` and has the same - signature. This slot may be left to *NULL*, in this case + signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it finds a match. @@ -2173,7 +2195,7 @@ Sequence Object Structures This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat` + may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` will fall back to :c:func:`PySequence_Concat`. It is also used by the augmented assignment ``+=``, after trying numeric in-place addition via the :c:member:`~PyNumberMethods.nb_inplace_add` slot. @@ -2182,7 +2204,7 @@ Sequence Object Structures This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat` + may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` will fall back to :c:func:`PySequence_Repeat`. It is also used by the augmented assignment ``*=``, after trying numeric in-place multiplication via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot. @@ -2214,7 +2236,7 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, - set :c:data:`view->obj` to *NULL* and return ``-1``. + set :c:data:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. @@ -2260,7 +2282,7 @@ Buffer Object Structures Handle a request to release the resources of the buffer. If no resources need to be released, :c:member:`PyBufferProcs.bf_releasebuffer` may be - *NULL*. Otherwise, a standard implementation of this function will take + ``NULL``. Otherwise, a standard implementation of this function will take these optional steps: (1) Decrement an internal counter for the number of exports. @@ -2314,7 +2336,7 @@ Async Object Structures The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must return ``1`` for it. - This slot may be set to *NULL* if an object is not an :term:`awaitable`. + This slot may be set to ``NULL`` if an object is not an :term:`awaitable`. .. c:member:: unaryfunc PyAsyncMethods.am_aiter @@ -2324,7 +2346,7 @@ Async Object Structures Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL* if an object does not implement + This slot may be set to ``NULL`` if an object does not implement asynchronous iteration protocol. .. c:member:: unaryfunc PyAsyncMethods.am_anext @@ -2334,7 +2356,7 @@ Async Object Structures PyObject *am_anext(PyObject *self); Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL*. + This slot may be set to ``NULL``. .. _slot-typedefs: @@ -2390,7 +2412,7 @@ Slot Type typedefs .. c:type:: int (*setattrfunc)(PyObject *self, char *attr, PyObject *value) Set the value of the named attribute for the object. - The value argument is set to *NULL* to delete the attribute. + The value argument is set to ``NULL`` to delete the attribute. .. c:type:: PyObject *(*getattrofunc)(PyObject *self, PyObject *attr) @@ -2401,7 +2423,7 @@ Slot Type typedefs .. c:type:: int (*setattrofunc)(PyObject *self, PyObject *attr, PyObject *value) Set the value of the named attribute for the object. - The value argument is set to *NULL* to delete the attribute. + The value argument is set to ``NULL`` to delete the attribute. See :c:member:`~PyTypeObject.tp_setattro`. @@ -2450,7 +2472,7 @@ Slot Type typedefs .. c:type:: int (*objobjargproc)(PyObject *, PyObject *, PyObject *) -.. _examples: +.. _typedef-examples: Examples ======== diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index a150a9c39d880c..6805ad48d96764 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -19,8 +19,7 @@ points must be below 1114112 (which is the full Unicode range). :c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated -and inefficient; it should be avoided in performance- or memory-sensitive -situations. +and inefficient. Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: @@ -34,6 +33,11 @@ can internally be in two states depending on how they were created: :c:type:`Py_UNICODE*` representation; you will have to call :c:func:`PyUnicode_READY` on them before calling any other API. +.. note:: + The "legacy" Unicode object will be removed in Python 3.12 with deprecated + APIs. All Unicode objects will be "canonical" since then. See :pep:`623` + for more information. + Unicode Type """""""""""" @@ -107,6 +111,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + This API will be removed with :c:func:`PyUnicode_FromUnicode`. + .. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) @@ -138,6 +145,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + ``PyUnicode_WCHAR_KIND`` is deprecated. + .. c:function:: int PyUnicode_KIND(PyObject *o) @@ -188,7 +198,7 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:macro:: PyUnicode_MAX_CHAR_VALUE(o) Return the maximum code point that is suitable for creating another string based on *o*, which must be in the "canonical" representation. This is @@ -208,7 +218,7 @@ access internal read-only data of Unicode objects: code units (this includes surrogate pairs as 2 units). *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -218,7 +228,7 @@ access internal read-only data of Unicode objects: Return the size of the deprecated :c:type:`Py_UNICODE` representation in bytes. *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -236,11 +246,11 @@ access internal read-only data of Unicode objects: .. versionchanged:: 3.3 This macro is now inefficient -- because in many cases the :c:type:`Py_UNICODE` representation does not exist and needs to be created - -- and can fail (return *NULL* with an exception set). Try to port the + -- and can fail (return ``NULL`` with an exception set). Try to port the code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using the :c:func:`PyUnicode_nBYTE_DATA` family of macros. @@ -413,12 +423,12 @@ APIs: Create a Unicode object from the char buffer *u*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new - object. If the buffer is not *NULL*, the return value might be a shared + object. If the buffer is not ``NULL``, the return value might be a shared object, i.e. modification of the data is not allowed. - If *u* is *NULL*, this function behaves like :c:func:`PyUnicode_FromUnicode` - with the buffer set to *NULL*. This usage is deprecated in favor of - :c:func:`PyUnicode_New`. + If *u* is ``NULL``, this function behaves like :c:func:`PyUnicode_FromUnicode` + with the buffer set to ``NULL``. This usage is deprecated in favor of + :c:func:`PyUnicode_New`, and will be removed in Python 3.12. .. c:function:: PyObject *PyUnicode_FromString(const char *u) @@ -443,82 +453,82 @@ APIs: .. tabularcolumns:: |l|l|L| - +-------------------+---------------------+--------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zd` | Py_ssize_t | Equivalent to | - | | | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zi` | Py_ssize_t | Equivalent to | - | | | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+--------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%U` | PyObject\* | A Unicode object. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%V` | PyObject\*, | A Unicode object (which may be | - | | const char\* | *NULL*) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | *NULL*). | - +-------------------+---------------------+--------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+--------------------------------+ + +-------------------+---------------------+----------------------------------+ + | Format Characters | Type | Comment | + +===================+=====================+==================================+ + | :attr:`%%` | *n/a* | The literal % character. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%c` | int | A single character, | + | | | represented as a C int. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%d` | int | Equivalent to | + | | | ``printf("%d")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%u` | unsigned int | Equivalent to | + | | | ``printf("%u")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%ld` | long | Equivalent to | + | | | ``printf("%ld")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%li` | long | Equivalent to | + | | | ``printf("%li")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lu` | unsigned long | Equivalent to | + | | | ``printf("%lu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lld` | long long | Equivalent to | + | | | ``printf("%lld")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%lli` | long long | Equivalent to | + | | | ``printf("%lli")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%llu` | unsigned long long | Equivalent to | + | | | ``printf("%llu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zd` | Py_ssize_t | Equivalent to | + | | | ``printf("%zd")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zi` | Py_ssize_t | Equivalent to | + | | | ``printf("%zi")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%zu` | size_t | Equivalent to | + | | | ``printf("%zu")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%i` | int | Equivalent to | + | | | ``printf("%i")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%x` | int | Equivalent to | + | | | ``printf("%x")``. [1]_ | + +-------------------+---------------------+----------------------------------+ + | :attr:`%s` | const char\* | A null-terminated C character | + | | | array. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%p` | const void\* | The hex representation of a C | + | | | pointer. Mostly equivalent to | + | | | ``printf("%p")`` except that | + | | | it is guaranteed to start with | + | | | the literal ``0x`` regardless | + | | | of what the platform's | + | | | ``printf`` yields. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%A` | PyObject\* | The result of calling | + | | | :func:`ascii`. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%U` | PyObject\* | A Unicode object. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%V` | PyObject\*, | A Unicode object (which may be | + | | const char\* | ``NULL``) and a null-terminated | + | | | C character array as a second | + | | | parameter (which will be used, | + | | | if the first parameter is | + | | | ``NULL``). | + +-------------------+---------------------+----------------------------------+ + | :attr:`%S` | PyObject\* | The result of calling | + | | | :c:func:`PyObject_Str`. | + +-------------------+---------------------+----------------------------------+ + | :attr:`%R` | PyObject\* | The result of calling | + | | | :c:func:`PyObject_Repr`. | + +-------------------+---------------------+----------------------------------+ An unrecognized format character causes all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. @@ -526,9 +536,9 @@ APIs: .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes for ``"%s"`` and - ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of + ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` - (if the ``PyObject*`` argument is not NULL). + (if the ``PyObject*`` argument is not ``NULL``). .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, zu, i, x): the 0-conversion flag has effect even when a precision is given. @@ -558,13 +568,13 @@ APIs: :class:`bytes`, :class:`bytearray` and other :term:`bytes-like objects ` are decoded according to the given *encoding* and using the error handling - defined by *errors*. Both can be *NULL* to have the interface use the default + defined by *errors*. Both can be ``NULL`` to have the interface use the default values (see :ref:`builtincodecs` for details). All other objects, including Unicode objects, cause a :exc:`TypeError` to be set. - The API returns *NULL* if there was an error. The caller is responsible for + The API returns ``NULL`` if there was an error. The caller is responsible for decref'ing the returned objects. @@ -640,7 +650,7 @@ APIs: Py_ssize_t buflen, int copy_null) Copy the string *u* into a UCS4 buffer, including a null character, if - *copy_null* is set. Returns *NULL* and sets an exception on error (in + *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of *u*). *buffer* is returned on success. @@ -650,7 +660,7 @@ APIs: .. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) Copy the string *u* into a new UCS4 buffer that is allocated using - :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a + :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -660,7 +670,7 @@ APIs: Deprecated Py_UNICODE APIs """""""""""""""""""""""""" -.. deprecated-removed:: 3.3 4.0 +.. deprecated-removed:: 3.3 3.12 These API functions are deprecated with the implementation of :pep:`393`. Extension modules can continue using them, as they will not be removed in Python @@ -670,42 +680,51 @@ Extension modules can continue using them, as they will not be removed in Python .. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u* - may be *NULL* which causes the contents to be undefined. It is the user's + may be ``NULL`` which causes the contents to be undefined. It is the user's responsibility to fill in the needed data. The buffer is copied into the new object. - If the buffer is not *NULL*, the return value might be a shared object. + If the buffer is not ``NULL``, the return value might be a shared object. Therefore, modification of the resulting Unicode object is only allowed when - *u* is *NULL*. + *u* is ``NULL``. - If the buffer is *NULL*, :c:func:`PyUnicode_READY` must be called once the + If the buffer is ``NULL``, :c:func:`PyUnicode_READY` must be called once the string content has been filled before using any of the access macros such as :c:func:`PyUnicode_KIND`. - Please migrate to using :c:func:`PyUnicode_FromKindAndData`, - :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_New`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_FromKindAndData`, :c:func:`PyUnicode_FromWideChar`, or + :c:func:`PyUnicode_New`. .. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) Return a read-only pointer to the Unicode object's internal - :c:type:`Py_UNICODE` buffer, or *NULL* on error. This will create the + :c:type:`Py_UNICODE` buffer, or ``NULL`` on error. This will create the :c:type:`Py_UNICODE*` representation of the object if it is not yet available. The buffer is always terminated with an extra null code point. Note that the resulting :c:type:`Py_UNICODE` string may also contain embedded null code points, which would cause the string to be truncated when used in most C functions. - Please migrate to using :c:func:`PyUnicode_AsUCS4`, - :c:func:`PyUnicode_AsWideChar`, :c:func:`PyUnicode_ReadChar` or similar new - APIs. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. + + .. deprecated-removed:: 3.3 3.10 .. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size) Create a Unicode object by replacing all decimal digits in :c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9 - according to their decimal value. Return *NULL* if an exception occurs. + according to their decimal value. Return ``NULL`` if an exception occurs. + + .. deprecated-removed:: 3.3 3.11 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`Py_UNICODE_TODECIMAL`. .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) @@ -718,10 +737,15 @@ Extension modules can continue using them, as they will not be removed in Python .. versionadded:: 3.3 + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. + .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode) - Create a copy of a Unicode string ending with a null code point. Return *NULL* + Create a copy of a Unicode string ending with a null code point. Return ``NULL`` and raise a :exc:`MemoryError` exception on memory allocation failure, otherwise return a new allocated buffer (use :c:func:`PyMem_Free` to free the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may @@ -738,7 +762,9 @@ Extension modules can continue using them, as they will not be removed in Python Return the size of the deprecated :c:type:`Py_UNICODE` representation, in code units (this includes surrogate pairs as 2 units). - Please migrate to using :c:func:`PyUnicode_GetLength`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_GET_LENGTH`. .. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj) @@ -932,7 +958,7 @@ wchar_t Support Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, using wcslen. - Return *NULL* on failure. + Return ``NULL`` on failure. .. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) @@ -951,22 +977,22 @@ wchar_t Support .. c:function:: wchar_t* PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size) Convert the Unicode object to a wide character string. The output string - always ends with a null character. If *size* is not *NULL*, write the number + always ends with a null character. If *size* is not ``NULL``, write the number of wide characters (excluding the trailing null termination character) into *\*size*. Note that the resulting :c:type:`wchar_t` string might contain null characters, which would cause the string to be truncated when used with - most C functions. If *size* is *NULL* and the :c:type:`wchar_t*` string + most C functions. If *size* is ``NULL`` and the :c:type:`wchar_t*` string contains null characters a :exc:`ValueError` is raised. Returns a buffer allocated by :c:func:`PyMem_Alloc` (use - :c:func:`PyMem_Free` to free it) on success. On error, returns *NULL* + :c:func:`PyMem_Free` to free it) on success. On error, returns ``NULL`` and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation is failed. .. versionadded:: 3.2 .. versionchanged:: 3.7 - Raises a :exc:`ValueError` if *size* is *NULL* and the :c:type:`wchar_t*` + Raises a :exc:`ValueError` if *size* is ``NULL`` and the :c:type:`wchar_t*` string contains null characters. @@ -982,7 +1008,7 @@ Many of the following APIs take two arguments encoding and errors, and they have the same semantics as the ones of the built-in :func:`str` string object constructor. -Setting encoding to *NULL* causes the default encoding to be used +Setting encoding to ``NULL`` causes the default encoding to be used which is ASCII. The file system calls should use :c:func:`PyUnicode_FSConverter` for encoding file names. This uses the variable :c:data:`Py_FileSystemDefaultEncoding` internally. This @@ -990,7 +1016,7 @@ variable should be treated as read-only: on some systems, it will be a pointer to a static string, on others, it will change at run-time (such as when the application invokes setlocale). -Error handling is set by errors which may also be set to *NULL* meaning to use +Error handling is set by errors which may also be set to ``NULL`` meaning to use the default handling defined for the codec. Default error handling for all built-in codecs is "strict" (:exc:`ValueError` is raised). @@ -1010,7 +1036,7 @@ These are the generic codec APIs: Create a Unicode object by decoding *size* bytes of the encoded string *s*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by + using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. @@ -1020,7 +1046,7 @@ These are the generic codec APIs: Encode a Unicode object and return the result as Python bytes object. *encoding* and *errors* have the same meaning as the parameters of the same name in the Unicode :meth:`~str.encode` method. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by + using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. @@ -1030,10 +1056,10 @@ These are the generic codec APIs: Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python bytes object. *encoding* and *errors* have the same meaning as the parameters of the same name in the Unicode :meth:`~str.encode` method. The codec - to be used is looked up using the Python codec registry. Return *NULL* if an + to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1047,14 +1073,14 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF8`. If - *consumed* is not *NULL*, trailing incomplete UTF-8 byte sequences will not be + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If + *consumed* is not ``NULL``, trailing incomplete UTF-8 byte sequences will not be treated as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1062,7 +1088,7 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode) Encode a Unicode object using UTF-8 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. @@ -1070,11 +1096,11 @@ These are the UTF-8 codec APIs: Return a pointer to the UTF-8 encoding of the Unicode object, and store the size of the encoded representation (in bytes) in *size*. The - *size* argument can be *NULL*; in this case no size will be stored. The + *size* argument can be ``NULL``; in this case no size will be stored. The returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, *NULL* is returned with an exception set and no + In the case of an error, ``NULL`` is returned with an exception set and no *size* is stored. This caches the UTF-8 representation of the string in the Unicode object, and @@ -1100,10 +1126,10 @@ These are the UTF-8 codec APIs: .. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* using UTF-8 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF8String`, :c:func:`PyUnicode_AsUTF8AndSize` or :c:func:`PyUnicode_AsEncodedString`. @@ -1119,10 +1145,10 @@ These are the UTF-32 codec APIs: const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error + corresponding Unicode object. *errors* (if non-``NULL``) defines the error handling. It defaults to "strict". - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte + If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte order:: *byteorder == -1: little endian @@ -1137,16 +1163,16 @@ These are the UTF-32 codec APIs: After completion, *\*byteorder* is set to the current byte order at the end of input data. - If *byteorder* is *NULL*, the codec starts in native order mode. + If *byteorder* is ``NULL``, the codec starts in native order mode. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF32`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat trailing incomplete UTF-32 byte sequences (such as a number of bytes not divisible by four) as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1156,7 +1182,7 @@ These are the UTF-32 codec APIs: Return a Python byte string using the UTF-32 encoding in native byte order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1172,12 +1198,12 @@ These are the UTF-32 codec APIs: If byteorder is ``0``, the output string will always start with the Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is prepended. - If *Py_UNICODE_WIDE* is not defined, surrogate pairs will be output + If ``Py_UNICODE_WIDE`` is not defined, surrogate pairs will be output as a single code point. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF32String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1192,10 +1218,10 @@ These are the UTF-16 codec APIs: const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error + corresponding Unicode object. *errors* (if non-``NULL``) defines the error handling. It defaults to "strict". - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte + If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte order:: *byteorder == -1: little endian @@ -1211,16 +1237,16 @@ These are the UTF-16 codec APIs: After completion, *\*byteorder* is set to the current byte order at the end of input data. - If *byteorder* is *NULL*, the codec starts in native order mode. + If *byteorder* is ``NULL``, the codec starts in native order mode. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF16`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat trailing incomplete UTF-16 byte sequences (such as an odd number of bytes or a split surrogate pair) as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1230,7 +1256,7 @@ These are the UTF-16 codec APIs: Return a Python byte string using the UTF-16 encoding in native byte order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1246,13 +1272,13 @@ These are the UTF-16 codec APIs: If byteorder is ``0``, the output string will always start with the Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is prepended. - If *Py_UNICODE_WIDE* is defined, a single :c:type:`Py_UNICODE` value may get + If ``Py_UNICODE_WIDE`` is defined, a single :c:type:`Py_UNICODE` value may get represented as a surrogate pair. If it is not defined, each :c:type:`Py_UNICODE` values is interpreted as a UCS-2 character. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF16String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1266,14 +1292,14 @@ These are the UTF-7 codec APIs: .. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF7`. If - *consumed* is not *NULL*, trailing incomplete UTF-7 base-64 sections will not + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If + *consumed* is not ``NULL``, trailing incomplete UTF-7 base-64 sections will not be treated as an error. Those bytes will not be decoded and the number of bytes that have been decoded will be stored in *consumed*. @@ -1282,7 +1308,7 @@ These are the UTF-7 codec APIs: int base64SetO, int base64WhiteSpace, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given size using UTF-7 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. If *base64SetO* is nonzero, "Set O" (punctuation that has no otherwise @@ -1290,7 +1316,7 @@ These are the UTF-7 codec APIs: nonzero, whitespace will be encoded in base-64. Both are set to zero for the Python "utf-7" codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1305,22 +1331,22 @@ These are the "Unicode Escape" codec APIs: Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return *NULL* if an exception was raised by the codec. + string *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) Encode a Unicode object using Unicode-Escape and return the result as a - bytes object. Error handling is "strict". Return *NULL* if an exception was + bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and - return a bytes object. Return *NULL* if an exception was raised by the codec. + return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUnicodeEscapeString`. @@ -1335,13 +1361,13 @@ These are the "Raw Unicode Escape" codec APIs: Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return *NULL* if an exception was raised by the codec. + encoded string *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) Encode a Unicode object using Raw-Unicode-Escape and return the result as - a bytes object. Error handling is "strict". Return *NULL* if an exception + a bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. @@ -1349,9 +1375,9 @@ These are the "Raw Unicode Escape" codec APIs: Py_ssize_t size) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape - and return a bytes object. Return *NULL* if an exception was raised by the codec. + and return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsRawUnicodeEscapeString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1367,23 +1393,23 @@ ordinals and only these are accepted by the codecs during encoding. .. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) Encode a Unicode object using Latin-1 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeLatin1(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Latin-1 and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsLatin1String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1399,23 +1425,23 @@ codes generate errors. .. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return *NULL* if an exception was raised by the codec. + *s*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) Encode a Unicode object using ASCII and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeASCII(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using ASCII and - return a Python bytes object. Return *NULL* if an exception was raised by + return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsASCIIString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1436,10 +1462,10 @@ These are the mapping codec APIs: PyObject *mapping, const char *errors) Create a Unicode object by decoding *size* bytes of the encoded string *s* - using the given *mapping* object. Return *NULL* if an exception was raised + using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. - If *mapping* is *NULL*, Latin-1 decoding will be applied. Else + If *mapping* is ``NULL``, Latin-1 decoding will be applied. Else *mapping* must map bytes ordinals (integers in the range from 0 to 255) to Unicode strings, integers (which are then interpreted as Unicode ordinals) or ``None``. Unmapped data bytes -- ones which cause a @@ -1451,7 +1477,7 @@ These are the mapping codec APIs: .. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) Encode a Unicode object using the given *mapping* object and return the - result as a bytes object. Error handling is "strict". Return *NULL* if an + result as a bytes object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. The *mapping* object must map Unicode ordinal integers to bytes objects, @@ -1464,10 +1490,10 @@ These are the mapping codec APIs: PyObject *mapping, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return the result as a bytes object. Return *NULL* if + *mapping* object and return the result as a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsCharmapString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1475,17 +1501,21 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ - PyObject *mapping, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) - Translate a Unicode object using the given *mapping* object and return the - resulting Unicode object. Return *NULL* if an exception was raised by the + Translate a string by applying a character mapping table to it and return the + resulting Unicode object. Return ``NULL`` if an exception was raised by the codec. - The *mapping* object must map Unicode ordinal integers to Unicode strings, - integers (which are then interpreted as Unicode ordinals) or ``None`` - (causing deletion of the character). Unmapped character ordinals (ones - which cause a :exc:`LookupError`) are left untouched and are copied as-is. + The mapping table must map Unicode ordinal integers to Unicode ordinal integers + or ``None`` (causing deletion of the character). + + Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries + and sequences work well. Unmapped character ordinals (ones which cause a + :exc:`LookupError`) are left untouched and are copied as-is. + + *errors* has the usual meaning for codecs. It may be ``NULL`` which indicates to + use the default error handling. .. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1493,9 +1523,9 @@ The following codec API is special in that maps Unicode to Unicode. Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a character *mapping* table to it and return the resulting Unicode object. - Return *NULL* when an exception was raised by the codec. + Return ``NULL`` when an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API ` @@ -1512,14 +1542,14 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. - Return *NULL* if an exception was raised by the codec. + Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode + If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If + *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode trailing lead byte and the number of bytes that have been decoded will be stored in *consumed*. @@ -1527,14 +1557,14 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) Encode a Unicode object using MBCS and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was + object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_EncodeCodePage(int code_page, PyObject *unicode, const char *errors) Encode the Unicode object using the specified code page and return a Python - bytes object. Return *NULL* if an exception was raised by the codec. Use + bytes object. Return ``NULL`` if an exception was raised by the codec. Use :c:data:`CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 @@ -1543,7 +1573,7 @@ the user settings on the machine running the codec. .. c:function:: PyObject* PyUnicode_EncodeMBCS(const Py_UNICODE *s, Py_ssize_t size, const char *errors) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using MBCS and return - a Python bytes object. Return *NULL* if an exception was raised by the + a Python bytes object. Return ``NULL`` if an exception was raised by the codec. .. deprecated-removed:: 3.3 4.0 @@ -1565,7 +1595,7 @@ The following APIs are capable of handling Unicode objects and strings on input (we refer to them as strings in the descriptions) and return Unicode objects or integers as appropriate. -They all return *NULL* or ``-1`` if an exception occurs. +They all return ``NULL`` or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_Concat(PyObject *left, PyObject *right) @@ -1575,7 +1605,7 @@ They all return *NULL* or ``-1`` if an exception occurs. .. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) - Split a string giving a list of Unicode strings. If *sep* is *NULL*, splitting + Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given separator. At most *maxsplit* splits will be done. If negative, no limit is set. Separators are not included in the resulting list. @@ -1588,23 +1618,6 @@ They all return *NULL* or ``-1`` if an exception occurs. characters are not included in the resulting strings. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \ - const char *errors) - - Translate a string by applying a character mapping table to it and return the - resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode ordinal integers - or ``None`` (causing deletion of the character). - - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. - - *errors* has the usual meaning for codecs. It may be *NULL* which indicates to - use the default error handling. - - .. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) Join a sequence of strings using the given *separator* and return the resulting diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 3fe0ae47aac35a..208a14e62a0e39 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -16,11 +16,11 @@ parameter. The available start symbols are :const:`Py_eval_input`, :const:`Py_file_input`, and :const:`Py_single_input`. These are described following the functions which accept them as parameters. -Note also that several of these functions take :c:type:`FILE\*` parameters. One +Note also that several of these functions take :c:type:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` structure for different C libraries can be different and incompatible. Under Windows (at least), it is possible for dynamically linked extensions to actually -use different libraries, so care should be taken that :c:type:`FILE\*` parameters +use different libraries, so care should be taken that :c:type:`FILE*` parameters are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. @@ -52,7 +52,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. + *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: int PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -64,7 +64,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *flags* argument set to *NULL*. + the *flags* argument set to ``NULL``. .. c:function:: int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -73,14 +73,14 @@ the same library that the Python runtime is using. terminal input or Unix pseudo-terminal), return the value of :c:func:`PyRun_InteractiveLoop`, otherwise return the result of :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this + encoding (:func:`sys.getfilesystemencoding`). If *filename* is ``NULL``, this function uses ``"???"`` as the filename. .. c:function:: int PyRun_SimpleString(const char *command) This is a simplified interface to :c:func:`PyRun_SimpleStringFlags` below, - leaving the *PyCompilerFlags\** argument set to NULL. + leaving the :c:type:`PyCompilerFlags`\* argument set to ``NULL``. .. c:function:: int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) @@ -99,13 +99,13 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_SimpleFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *closeit* set to ``0`` and *flags* set to *NULL*. + leaving *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) @@ -124,7 +124,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -144,7 +144,7 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, - leaving *flags* set to *NULL*. + leaving *flags* set to ``NULL``. .. c:function:: int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) @@ -173,13 +173,13 @@ the same library that the Python runtime is using. ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, overriding the default function used to read a single line of input at the interpreter's prompt. The function is expected to output - the string *prompt* if it's not *NULL*, and then read a line of + the string *prompt* if it's not ``NULL``, and then read a line of input from the provided standard input file, returning the resulting string. For example, The :mod:`readline` module sets this hook to provide line-editing and tab-completion features. The result must be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. + :c:func:`PyMem_RawRealloc`, or ``NULL`` if an error occurred. .. versionchanged:: 3.4 The result must be allocated by :c:func:`PyMem_RawMalloc` or @@ -191,14 +191,14 @@ the same library that the Python runtime is using. This is a simplified interface to :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL* and *flags* set to ``0``. + to ``NULL`` and *flags* set to ``0``. .. c:function:: struct _node* PyParser_SimpleParseStringFlags( const char *str, int start, int flags) This is a simplified interface to :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL*. + to ``NULL``. .. c:function:: struct _node* PyParser_SimpleParseStringFlagsFilename( const char *str, const char *filename, int start, int flags) @@ -225,7 +225,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -236,20 +236,20 @@ the same library that the Python runtime is using. that implements the mapping protocol. The parameter *start* specifies the start token that should be used to parse the source code. - Returns the result of executing the code as a Python object, or *NULL* if an + Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. .. c:function:: PyObject* PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. + *closeit* set to ``0`` and *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit) This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) @@ -270,7 +270,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileString(const char *str, const char *filename, int start) This is a simplified interface to :c:func:`Py_CompileStringFlags` below, leaving - *flags* set to *NULL*. + *flags* set to ``NULL``. .. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) @@ -286,7 +286,7 @@ the same library that the Python runtime is using. code which can be compiled and should be :const:`Py_eval_input`, :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns *NULL* if the code + :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. The integer *optimize* specifies the optimization level of the compiler; a @@ -309,7 +309,7 @@ the same library that the Python runtime is using. This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just the code object, and global and local variables. The other arguments are - set to *NULL*. + set to ``NULL``. .. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, PyObject *const *kws, int kwcount, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure) @@ -335,12 +335,12 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - This is the main, unvarnished function of Python interpretation. It is - literally 2000 lines long. The code object associated with the execution - frame *f* is executed, interpreting bytecode and executing calls as needed. - The additional *throwflag* parameter can mostly be ignored - if true, then - it causes an exception to immediately be thrown; this is used for the - :meth:`~generator.throw` methods of generator objects. + This is the main, unvarnished function of Python interpretation. The code + object associated with the execution frame *f* is executed, interpreting + bytecode and executing calls as needed. The additional *throwflag* + parameter can mostly be ignored - if true, then it causes an exception + to immediately be thrown; this is used for the :meth:`~generator.throw` + methods of generator objects. .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it @@ -386,13 +386,24 @@ the same library that the Python runtime is using. executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from __future__ import`` can modify *flags*. - Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as + Whenever ``PyCompilerFlags *flags`` is ``NULL``, :attr:`cf_flags` is treated as equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: + discarded. - struct PyCompilerFlags { - int cf_flags; - } + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in *cf_flags*. + + .. versionchanged:: 3.8 + Added *cf_feature_version* field. .. c:var:: int CO_FUTURE_DIVISION diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 7a4f8615b9c32b..e3a9bda54d671a 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -33,9 +33,9 @@ as much as it can. reference object may be returned. The second parameter, *callback*, can be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object - itself. *callback* may also be ``None`` or *NULL*. If *ob* is not a + itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly-referencable object, or if *callback* is not callable, ``None``, or - *NULL*, this will return *NULL* and raise :exc:`TypeError`. + ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -45,9 +45,9 @@ as much as it can. existing proxy object may be returned. The second parameter, *callback*, can be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak - reference object itself. *callback* may also be ``None`` or *NULL*. If *ob* + reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly-referencable object, or if *callback* is not callable, - ``None``, or *NULL*, this will return *NULL* and raise :exc:`TypeError`. + ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. .. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) diff --git a/Doc/conf.py b/Doc/conf.py index e85ea5b2d2ff49..4cca13b119374c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -48,8 +48,10 @@ needs_sphinx = '1.8' # Ignore any .rst files in the venv/ directory. -venvdir = os.getenv('VENVDIR', 'venv') -exclude_patterns = [venvdir+'/*', 'README.rst'] +exclude_patterns = ['venv/*', 'README.rst'] +venvdir = os.getenv('VENVDIR') +if venvdir is not None: + exclude_patterns.append(venvdir + '/*') # Disable Docutils smartquotes for several translations smartquotes_excludes = { @@ -224,3 +226,13 @@ # Relative filename of the reference count data file. refcount_file = 'data/refcounts.dat' + +# Sphinx 2 and Sphinx 3 compatibility +# ----------------------------------- + +# bpo-40204: Allow Sphinx 2 syntax in the C domain +c_allow_pre_v3 = True + +# bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the +# documentation is built with -W (warnings treated as errors). +c_warn_on_allowed_pre_v3 = False diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 393a1f03751f82..9b71683155eebe 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2019 Python Software Foundation. All rights reserved. +Copyright © 2001-2023 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/python3.8.abi b/Doc/data/python3.8.abi new file mode 100644 index 00000000000000..90b2b8b660cade --- /dev/null +++ b/Doc/data/python3.8.abi @@ -0,0 +1,16020 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/data/python3.8.abi.ignorefile b/Doc/data/python3.8.abi.ignorefile new file mode 100644 index 00000000000000..a9c89f0a746c83 --- /dev/null +++ b/Doc/data/python3.8.abi.ignorefile @@ -0,0 +1,8 @@ +[suppress_type] +name_regexp = _Py.* + +[suppress_variable] +name_regexp = _Py.* + +[suppress_function] +name_regexp = _Py.* diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index aca57a1dae9d8c..3ccc20b70c62e6 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -234,9 +234,26 @@ PyCode_Check:PyObject*:co:0: PyCode_GetNumFree:int::: PyCode_GetNumFree:PyCodeObject*:co:0: +PyCode_NewWithPosOnlyArgs:PyCodeObject*::+1: +PyCode_NewWithPosOnlyArgs:int:argcount:: +PyCode_NewWithPosOnlyArgs:int:posonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:kwonlyargcount:: +PyCode_NewWithPosOnlyArgs:int:nlocals:: +PyCode_NewWithPosOnlyArgs:int:stacksize:: +PyCode_NewWithPosOnlyArgs:int:flags:: +PyCode_NewWithPosOnlyArgs:PyObject*:code:0: +PyCode_NewWithPosOnlyArgs:PyObject*:consts:0: +PyCode_NewWithPosOnlyArgs:PyObject*:names:0: +PyCode_NewWithPosOnlyArgs:PyObject*:varnames:0: +PyCode_NewWithPosOnlyArgs:PyObject*:freevars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:cellvars:0: +PyCode_NewWithPosOnlyArgs:PyObject*:filename:0: +PyCode_NewWithPosOnlyArgs:PyObject*:name:0: +PyCode_NewWithPosOnlyArgs:int:firstlineno:: +PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0: + PyCode_New:PyCodeObject*::+1: PyCode_New:int:argcount:: -PyCode_New:int:posonlyargcount:: PyCode_New:int:kwonlyargcount:: PyCode_New:int:nlocals:: PyCode_New:int:stacksize:: @@ -1656,7 +1673,8 @@ PyObject_GenericSetAttr:PyObject*:name:0: PyObject_GenericSetAttr:PyObject*:value:+1: PyObject_GenericSetDict:int::: -PyObject_GenericSetDict:PyObject*:o:+1: +PyObject_GenericSetDict:PyObject*:o:0: +PyObject_GenericSetDict:PyObject*:value:+1: PyObject_GenericSetDict:void*:context:: PyObject_GetAttr:PyObject*::+1: @@ -2445,7 +2463,7 @@ PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: PyUnicode_AsWideChar:PyObject*:*unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:Pyssize_t:size:: +PyUnicode_AsWideChar:Py_ssize_t:size:: PyUnicode_AsWideCharString:wchar_t*::: PyUnicode_AsWideCharString:PyObject*:unicode:0: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 2e46c7ac8635e8..02379946244d84 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -40,7 +40,7 @@ Key terms evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation and issue trackers on both `GitHub `__ and - `BitBucket `__. + `Bitbucket `__. * :mod:`distutils` is the original build and distribution system first added to the Python standard library in 1998. While direct use of :mod:`distutils` is being phased out, it still laid the foundation for the current packaging @@ -128,6 +128,7 @@ involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ * `Uploading the project to the Python Packaging Index`_ +* `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/distributing-packages/ @@ -135,6 +136,8 @@ involved in creating and publishing a project: https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project .. _Uploading the project to the Python Packaging Index: \ https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi +.. _The .pypirc file: \ + https://packaging.python.org/specifications/pypirc/ How do I...? @@ -148,7 +151,7 @@ These are quick answers or links for some common tasks. This isn't an easy topic, but here are a few tips: * check the Python Packaging Index to see if the name is already in use -* check popular hosting sites like GitHub, BitBucket, etc to see if there +* check popular hosting sites like GitHub, Bitbucket, etc to see if there is already a project with that name * check what comes up in a web search for the name you're considering * avoid particularly common words, especially ones with multiple meanings, diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 2601d30f63eb4c..80136b8a6bcdd8 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -6,12 +6,12 @@ API Reference .. seealso:: - `New and changed setup.py arguments in setuptools `_ + `New and changed setup.py arguments in setuptools`_ The ``setuptools`` project adds new capabilities to the ``setup`` function and other APIs, makes the API consistent across different Python versions, and is hence recommended over using ``distutils`` directly. -.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords +.. _New and changed setup.py arguments in setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords .. include:: ./_setuptools_disclaimer.rst @@ -1545,7 +1545,7 @@ Python's own build procedures. ================================================= .. module:: distutils.text_file - :synopsis: provides the TextFile class, a simple interface to text files + :synopsis: Provides the TextFile class, a simple interface to text files This module provides the :class:`TextFile` class, which gives an interface to @@ -1684,7 +1684,7 @@ lines, and joining lines with backslashes. =================================================== .. module:: distutils.version - :synopsis: implements classes that represent module version numbers. + :synopsis: Implements classes that represent module version numbers. .. % todo @@ -1699,7 +1699,7 @@ lines, and joining lines with backslashes. =================================================================== .. module:: distutils.cmd - :synopsis: This module provides the abstract base class Command. This class + :synopsis: Provides the abstract base class :class:`~distutils.cmd.Command`. This class is subclassed by the modules in the distutils.command subpackage. @@ -1792,7 +1792,7 @@ Subclasses of :class:`Command` must define the following methods. ========================================================== .. module:: distutils.command - :synopsis: This subpackage contains one module for each standard Distutils command. + :synopsis: Contains one module for each standard Distutils command. .. % \subsubsection{Individual Distutils commands} @@ -1863,6 +1863,9 @@ Subclasses of :class:`Command` must define the following methods. .. module:: distutils.command.bdist_wininst :synopsis: Build a Windows installer +.. deprecated:: 3.8 + Use bdist_wheel (wheel packages) instead. + .. % todo @@ -2036,7 +2039,7 @@ This is described in more detail in :pep:`301`. =================================================================== .. module:: distutils.command.check - :synopsis: Check the metadata of a package + :synopsis: Check the meta-data of a package The ``check`` command performs some tests on the meta-data of a package. diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index f44d0d039f45f4..b814f2e9508c9c 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -146,6 +146,9 @@ generated by each, are: | :command:`bdist_msi` | msi | +--------------------------+-------------------------------------+ +.. note:: + bdist_wininst is deprecated since Python 3.8. + The following sections give details on the individual :command:`bdist_\*` commands. @@ -298,6 +301,9 @@ file winds up deep in the "build tree," in a temporary directory created by Creating Windows Installers =========================== +.. warning:: + bdist_wininst is deprecated since Python 3.8. + Executable installers are the natural format for binary distributions on Windows. They display a nice graphical user interface, display some information about the module distribution to be installed taken from the metadata in the @@ -315,8 +321,8 @@ or the :command:`bdist` command with the :option:`!--formats` option:: If you have a pure module distribution (only containing pure Python modules and packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix -platforms or Mac OS X. +like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary +distributions in only supported on Windows systems. If you have a non-pure distribution, the extensions can only be created on a Windows platform, and will be Python version dependent. The installer filename @@ -459,3 +465,6 @@ Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-contr option. The default is 'none' (meaning no UAC handling is done), and other valid values are 'auto' (meaning prompt for UAC elevation if Python was installed for all users) and 'force' (meaning always prompt for elevation). + +.. note:: + bdist_wininst is deprecated since Python 3.8. diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst index 4ac552c7c6997a..44f70831d0bf87 100644 --- a/Doc/distutils/examples.rst +++ b/Doc/distutils/examples.rst @@ -1,8 +1,8 @@ .. _distutils_examples: -******** -Examples -******** +****************** +Distutils Examples +****************** .. include:: ./_setuptools_disclaimer.rst diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 483bc852f6c777..5f5abdf9c15067 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -196,7 +196,7 @@ function is then made with:: pValue = PyObject_CallObject(pFunc, pArgs); -Upon return of the function, ``pValue`` is either *NULL* or it contains a +Upon return of the function, ``pValue`` is either ``NULL`` or it contains a reference to the return value of the function. Be sure to release the reference after examining the value. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index e459514b2b5853..bc85a05ff2f850 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -117,7 +117,7 @@ store the converted values. More about this later. type and its components have been stored in the variables whose addresses are passed. It returns false (zero) if an invalid argument list was passed. In the latter case it also raises an appropriate exception so the calling function can -return *NULL* immediately (as we saw in the example). +return ``NULL`` immediately (as we saw in the example). .. _extending-errors: @@ -127,8 +127,8 @@ Intermezzo: Errors and Exceptions An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value -(usually a *NULL* pointer). Exceptions are stored in a static global variable -inside the interpreter; if this variable is *NULL* no exception has occurred. A +(usually a ``NULL`` pointer). Exceptions are stored in a static global variable +inside the interpreter; if this variable is ``NULL`` no exception has occurred. A second global variable stores the "associated value" of the exception (the second argument to :keyword:`raise`). A third variable contains the stack traceback in case the error originated in Python code. These three variables @@ -152,13 +152,13 @@ its associated value. You don't need to :c:func:`Py_INCREF` the objects passed to any of these functions. You can test non-destructively whether an exception has been set with -:c:func:`PyErr_Occurred`. This returns the current exception object, or *NULL* +:c:func:`PyErr_Occurred`. This returns the current exception object, or ``NULL`` if no exception has occurred. You normally don't need to call :c:func:`PyErr_Occurred` to see whether an error occurred in a function call, since you should be able to tell from the return value. When a function *f* that calls another function *g* detects that the latter -fails, *f* should itself return an error value (usually *NULL* or ``-1``). It +fails, *f* should itself return an error value (usually ``NULL`` or ``-1``). It should *not* call one of the :c:func:`PyErr_\*` functions --- one has already been called by *g*. *f*'s caller is then supposed to also return an error indication to *its* caller, again *without* calling :c:func:`PyErr_\*`, and so on @@ -209,7 +209,7 @@ usually declare a static object variable at the beginning of your file:: static PyObject *SpamError; and initialize it in your module's initialization function (:c:func:`PyInit_spam`) -with an exception object (leaving out the error checking for now):: +with an exception object:: PyMODINIT_FUNC PyInit_spam(void) @@ -221,14 +221,20 @@ with an exception object (leaving out the error checking for now):: return NULL; SpamError = PyErr_NewException("spam.error", NULL, NULL); - Py_INCREF(SpamError); - PyModule_AddObject(m, "error", SpamError); + Py_XINCREF(SpamError); + if (PyModule_AddObject(m, "error", SpamError) < 0) { + Py_XDECREF(SpamError); + Py_CLEAR(SpamError); + Py_DECREF(m); + return NULL; + } + return m; } Note that the Python name for the exception object is :exc:`spam.error`. The :c:func:`PyErr_NewException` function may create a class with the base class -being :exc:`Exception` (unless another class is passed in instead of *NULL*), +being :exc:`Exception` (unless another class is passed in instead of ``NULL``), described in :ref:`bltin-exceptions`. Note also that the :c:data:`SpamError` variable retains a reference to the newly @@ -272,7 +278,7 @@ statement:: if (!PyArg_ParseTuple(args, "s", &command)) return NULL; -It returns *NULL* (the error indicator for functions returning object pointers) +It returns ``NULL`` (the error indicator for functions returning object pointers) if an error is detected in the argument list, relying on the exception set by :c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been copied to the local variable :c:data:`command`. This is a pointer assignment and @@ -302,7 +308,7 @@ macro):: return Py_None; :c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a *NULL* pointer, which means "error" in most +genuine Python object rather than a ``NULL`` pointer, which means "error" in most contexts, as we have seen. @@ -370,7 +376,7 @@ inserts built-in function objects into the newly created module based upon the table (an array of :c:type:`PyMethodDef` structures) found in the module definition. :c:func:`PyModule_Create` returns a pointer to the module object that it creates. It may abort with a fatal error for -certain errors, or return *NULL* if the module could not be initialized +certain errors, or return ``NULL`` if the module could not be initialized satisfactorily. The init function must return the module object to its caller, so that it then gets inserted into ``sys.modules``. @@ -389,18 +395,26 @@ optionally followed by an import of the module:: } /* Add a built-in module, before Py_Initialize */ - PyImport_AppendInittab("spam", PyInit_spam); + if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { + fprintf(stderr, "Error: could not extend in-built modules table\n"); + exit(1); + } /* Pass argv[0] to the Python interpreter */ Py_SetProgramName(program); - /* Initialize the Python interpreter. Required. */ + /* Initialize the Python interpreter. Required. + If this step fails, it will be a fatal error. */ Py_Initialize(); /* Optionally import the module; alternatively, import can be deferred until the embedded script imports it. */ - PyImport_ImportModule("spam"); + PyObject *pmodule = PyImport_ImportModule("spam"); + if (!pmodule) { + PyErr_Print(); + fprintf(stderr, "Error: could not import module 'spam'\n"); + } ... @@ -520,8 +534,8 @@ This function must be registered with the interpreter using the :ref:`parsetuple`. The macros :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` increment/decrement the -reference count of an object and are safe in the presence of *NULL* pointers -(but note that *temp* will not be *NULL* in this context). More info on them +reference count of an object and are safe in the presence of ``NULL`` pointers +(but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. .. index:: single: PyObject_CallObject() @@ -530,7 +544,7 @@ Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to arbitrary Python objects: the Python function, and the argument list. The argument list must always be a tuple object, whose length is the number of -arguments. To call the Python function with no arguments, pass in NULL, or +arguments. To call the Python function with no arguments, pass in ``NULL``, or an empty tuple; to call it with one argument, pass a singleton tuple. :c:func:`Py_BuildValue` returns a tuple when its format string consists of zero or more format codes between parentheses. For example:: @@ -560,7 +574,7 @@ somehow :c:func:`Py_DECREF` the result, even (especially!) if you are not interested in its value. Before you do this, however, it is important to check that the return value -isn't *NULL*. If it is, the Python function terminated by raising an exception. +isn't ``NULL``. If it is, the Python function terminated by raising an exception. If the C code that called :c:func:`PyObject_CallObject` is called from Python, it should now return an error indication to its Python caller, so the interpreter can print a stack trace, or the calling Python code can handle the exception. @@ -717,7 +731,7 @@ The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of keywords received as the third parameter from the Python runtime. The *kwlist* -parameter is a *NULL*-terminated list of strings which identify the parameters; +parameter is a ``NULL``-terminated list of strings which identify the parameters; the names are matched with the type information from *format* from left to right. On success, :c:func:`PyArg_ParseTupleAndKeywords` returns true, otherwise it returns false and raises an appropriate exception. @@ -1078,32 +1092,32 @@ NULL Pointers ------------- In general, functions that take object references as arguments do not expect you -to pass them *NULL* pointers, and will dump core (or cause later core dumps) if -you do so. Functions that return object references generally return *NULL* only -to indicate that an exception occurred. The reason for not testing for *NULL* +to pass them ``NULL`` pointers, and will dump core (or cause later core dumps) if +you do so. Functions that return object references generally return ``NULL`` only +to indicate that an exception occurred. The reason for not testing for ``NULL`` arguments is that functions often pass the objects they receive on to other -function --- if each function were to test for *NULL*, there would be a lot of +function --- if each function were to test for ``NULL``, there would be a lot of redundant tests and the code would run more slowly. -It is better to test for *NULL* only at the "source:" when a pointer that may be -*NULL* is received, for example, from :c:func:`malloc` or from a function that +It is better to test for ``NULL`` only at the "source:" when a pointer that may be +``NULL`` is received, for example, from :c:func:`malloc` or from a function that may raise an exception. -The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for *NULL* +The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for ``NULL`` pointers --- however, their variants :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` do. The macros for checking for a particular object type (``Pytype_Check()``) don't -check for *NULL* pointers --- again, there is much code that calls several of +check for ``NULL`` pointers --- again, there is much code that calls several of these in a row to test an object against various different expected types, and -this would generate redundant tests. There are no variants with *NULL* +this would generate redundant tests. There are no variants with ``NULL`` checking. The C function calling mechanism guarantees that the argument list passed to C -functions (``args`` in the examples) is never *NULL* --- in fact it guarantees +functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees that it is always a tuple [#]_. -It is a severe error to ever let a *NULL* pointer "escape" to the Python user. +It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. .. Frank Stajano: A pedagogically buggy example, along the lines of the previous listing, would @@ -1178,7 +1192,7 @@ different ways between the module providing the code and the client modules. Whichever method you choose, it's important to name your Capsules properly. The function :c:func:`PyCapsule_New` takes a name parameter -(:c:type:`const char \*`); you're permitted to pass in a *NULL* name, but +(:c:type:`const char \*`); you're permitted to pass in a ``NULL`` name, but we strongly encourage you to specify a name. Properly named Capsules provide a degree of runtime type-safety; there is no feasible way to tell one unnamed Capsule from another. @@ -1261,8 +1275,12 @@ function must take care of initializing the C API pointer array:: /* Create a Capsule containing the API pointer array's address */ c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); - if (c_api_object != NULL) - PyModule_AddObject(m, "_C_API", c_api_object); + if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { + Py_XDECREF(c_api_object); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 308c06705e8d21..19c5e51537b192 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -191,7 +191,7 @@ For every object which can support attributes, the corresponding type must provide the functions that control how the attributes are resolved. There needs to be a function which can retrieve attributes (if any are defined), and another to set attributes (if setting attributes is allowed). Removing an attribute is -a special case, for which the new value passed to the handler is *NULL*. +a special case, for which the new value passed to the handler is ``NULL``. Python supports two pairs of attribute handlers; a type that supports attributes only needs to implement the functions for one pair. The difference is that one @@ -233,9 +233,9 @@ attributes, when the values are computed, or how relevant data is stored. When :c:func:`PyType_Ready` is called, it uses three tables referenced by the type object to create :term:`descriptor`\s which are placed in the dictionary of the type object. Each descriptor controls access to one attribute of the instance -object. Each of the tables is optional; if all three are *NULL*, instances of +object. Each of the tables is optional; if all three are ``NULL``, instances of the type will only have attributes that are inherited from their base type, and -should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as +should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields ``NULL`` as well, allowing the base type to handle attributes. The tables are declared as three fields of the type object:: @@ -244,7 +244,7 @@ The tables are declared as three fields of the type object:: struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; -If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of +If :c:member:`~PyTypeObject.tp_methods` is not ``NULL``, it must refer to an array of :c:type:`PyMethodDef` structures. Each entry in the table is an instance of this structure:: @@ -258,7 +258,7 @@ structure:: One entry should be defined for each method provided by the type; no entries are needed for methods inherited from a base type. One additional entry is needed at the end; it is a sentinel that marks the end of the array. The -:attr:`ml_name` field of the sentinel must be *NULL*. +:attr:`ml_name` field of the sentinel must be ``NULL``. The second table is used to define attributes which map directly to data stored in the instance. A variety of primitive C types are supported, and access may @@ -307,7 +307,7 @@ application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value -of *NULL* is required. +of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -352,9 +352,9 @@ Here is an example:: The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or :meth:`__delattr__` method of a class instance would be called. When an -attribute should be deleted, the third parameter will be *NULL*. Here is an +attribute should be deleted, the third parameter will be ``NULL``. Here is an example that simply raises an exception; if this were really all you wanted, the -:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: +:c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) @@ -380,7 +380,7 @@ where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GT``, ``Py_LT`` or ``Py_GT``. It should compare the two objects with respect to the specified operator and return ``Py_True`` or ``Py_False`` if the comparison is successful, ``Py_NotImplemented`` to indicate that comparison is not -implemented and the other object's comparison method should be tried, or *NULL* +implemented and the other object's comparison method should be tried, or ``NULL`` if an exception was set. Here is a sample implementation, for a datatype that is considered equal if the @@ -427,7 +427,7 @@ from the type implementation, the older protocols have been defined as optional blocks of handlers referenced by the type object. For newer protocols there are additional slots in the main type object, with a flag bit being set to indicate that the slots are present and should be checked by the interpreter. (The flag -bit does not indicate that the slot values are non-*NULL*. The flag may be set +bit does not indicate that the slot values are non-``NULL``. The flag may be set to indicate the presence of a slot, but a slot may still be unfilled.) :: PyNumberMethods *tp_as_number; @@ -478,9 +478,9 @@ This function takes three arguments: :c:func:`PyArg_ParseTuple` to extract the arguments. #. *kwds* is a dictionary of keyword arguments that were passed. If this is - non-*NULL* and you support keyword arguments, use + non-``NULL`` and you support keyword arguments, use :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you - do not want to support keyword arguments and this is non-*NULL*, raise a + do not want to support keyword arguments and this is non-``NULL``, raise a :exc:`TypeError` with a message saying that keyword arguments are not supported. Here is a toy ``tp_call`` implementation:: @@ -512,7 +512,7 @@ Here is a toy ``tp_call`` implementation:: These functions provide support for the iterator protocol. Both handlers take exactly one parameter, the instance for which they are being called, and return a new reference. In the case of an error, they should set an -exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds +exception and return ``NULL``. :c:member:`~PyTypeObject.tp_iter` corresponds to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` corresponds to the Python :meth:`~iterator.__next__` method. @@ -534,11 +534,11 @@ and :c:member:`~PyTypeObject.tp_iternext`. An iterator's to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should return a new reference to the next object in the iteration, if there is one. If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext` -may return *NULL* without setting an exception, or it may set -:exc:`StopIteration` *in addition* to returning *NULL*; avoiding +may return ``NULL`` without setting an exception, or it may set +:exc:`StopIteration` *in addition* to returning ``NULL``; avoiding the exception can yield slightly better performance. If an actual error occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception -and return *NULL*. +and return ``NULL``. .. _weakref-support: @@ -557,7 +557,7 @@ For an object to be weakly referencable, the extension type must do two things: #. Include a :c:type:`PyObject\*` field in the C object structure dedicated to the weak reference mechanism. The object's constructor should leave it - *NULL* (which is automatic when using the default + ``NULL`` (which is automatic when using the default :c:member:`~PyTypeObject.tp_alloc`). #. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member @@ -582,7 +582,7 @@ And the corresponding member in the statically-declared type object:: The only further addition is that ``tp_dealloc`` needs to clear any weak references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is -non-*NULL*:: +non-``NULL``:: static void Trivial_dealloc(TrivialObject *self) diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 59c8cc01222b2c..4da77e797d222c 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -177,9 +177,14 @@ Everything else in the file should be familiar, except for some code in This initializes the :class:`Custom` type, filling in a number of members to the appropriate default values, including :attr:`ob_type` that we initially -set to *NULL*. :: +set to ``NULL``. :: - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + Py_INCREF(&CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } This adds the type to the module dictionary. This allows us to create :class:`Custom` instances by calling the :class:`Custom` class: @@ -270,7 +275,7 @@ which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: This method first clears the reference counts of the two Python attributes. :c:func:`Py_XDECREF` correctly handles the case where its argument is -*NULL* (which might happen here if ``tp_new`` failed midway). It then +``NULL`` (which might happen here if ``tp_new`` failed midway). It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type (computed by ``Py_TYPE(self)``) to free the object's memory. Note that the object's type might not be :class:`CustomType`, because the object may @@ -316,7 +321,7 @@ objects of the type. It is exposed in Python as the :meth:`__new__` method. It is not required to define a ``tp_new`` member, and indeed many extension types will simply reuse :c:func:`PyType_GenericNew` as done in the first version of the ``Custom`` type above. In this case, we use the ``tp_new`` -handler to initialize the ``first`` and ``last`` attributes to non-*NULL* +handler to initialize the ``first`` and ``last`` attributes to non-``NULL`` default values. ``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``, @@ -336,7 +341,7 @@ slot to allocate memory:: self = (CustomObject *) type->tp_alloc(type, 0); Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc` -result against *NULL* before proceeding. +result against ``NULL`` before proceeding. .. note:: We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather @@ -411,7 +416,7 @@ But this would be risky. Our type doesn't restrict the type of the ``first`` member, so it could be any kind of object. It could have a destructor that causes code to be executed that tries to access the ``first`` member; or that destructor could release the -:term:`Global interpreter Lock` and let arbitrary code run in other +:term:`Global interpreter Lock ` and let arbitrary code run in other threads that accesses and modifies our object. To be paranoid and protect ourselves against this possibility, we almost @@ -450,9 +455,9 @@ below for details. A disadvantage of this approach is that it doesn't provide a way to restrict the types of objects that can be assigned to the Python attributes. We expect the first and last names to be strings, but any Python objects can be assigned. -Further, the attributes can be deleted, setting the C pointers to *NULL*. Even -though we can make sure the members are initialized to non-*NULL* values, the -members can be set to *NULL* if the attributes are deleted. +Further, the attributes can be deleted, setting the C pointers to ``NULL``. Even +though we can make sure the members are initialized to non-``NULL`` values, the +members can be set to ``NULL`` if the attributes are deleted. We define a single method, :meth:`Custom.name()`, that outputs the objects name as the concatenation of the first and last names. :: @@ -484,8 +489,8 @@ equivalent to the Python method: return "%s %s" % (self.first, self.last) Note that we have to check for the possibility that our :attr:`first` and -:attr:`last` members are *NULL*. This is because they can be deleted, in which -case they are set to *NULL*. It would be better to prevent deletion of these +:attr:`last` members are ``NULL``. This is because they can be deleted, in which +case they are set to ``NULL``. It would be better to prevent deletion of these attributes and to restrict the attribute values to be strings. We'll see how to do that in the next section. @@ -579,7 +584,7 @@ could, for example, be used to allow a single set of getter and setter functions that decide the attribute to get or set based on data in the closure.) The setter function is passed the :class:`Custom` object, the new value, and the -closure. The new value may be *NULL*, in which case the attribute is being +closure. The new value may be ``NULL``, in which case the attribute is being deleted. In our setter, we raise an error if the attribute is deleted or if its new value is not a string. @@ -598,7 +603,7 @@ and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: .tp_getset = Custom_getsetters, The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned -above. In this case, we aren't using a closure, so we just pass *NULL*. +above. In this case, we aren't using a closure, so we just pass ``NULL``. We also remove the member definitions for these attributes:: @@ -638,7 +643,7 @@ allow strings [#]_ to be passed:: } With these changes, we can assure that the ``first`` and ``last`` members are -never *NULL* so we can remove checks for *NULL* values in almost all cases. +never ``NULL`` so we can remove checks for ``NULL`` values in almost all cases. This means that most of the :c:func:`Py_XDECREF` calls can be converted to :c:func:`Py_DECREF` calls. The only place we can't change these calls is in the ``tp_dealloc`` implementation, where there is the possibility that the @@ -744,7 +749,7 @@ participate in cycles:: Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe way to clear data attributes of arbitrary types while decrementing their reference counts. If you were to call :c:func:`Py_XDECREF` instead -on the attribute before setting it to *NULL*, there is a possibility +on the attribute before setting it to ``NULL``, there is a possibility that the attribute's destructor would call back into code that reads the attribute again (*especially* if there is a reference cycle). @@ -864,7 +869,12 @@ function:: return NULL; Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2d63a0323da66..0d4c8d8fd27ae3 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -146,69 +146,20 @@ variables and instance variables live in two different namespaces, and you need to tell Python which namespace to use. +.. _why-can-t-i-use-an-assignment-in-an-expression: + Why can't I use an assignment in an expression? ----------------------------------------------- -Many people used to C or Perl complain that they want to use this C idiom: - -.. code-block:: c - - while (line = readline(f)) { - // do something with line - } - -where in Python you're forced to write this:: - - while True: - line = f.readline() - if not line: - break - ... # do something with line - -The reason for not allowing assignment in Python expressions is a common, -hard-to-find bug in those other languages, caused by this construct: - -.. code-block:: c +Starting in Python 3.8, you can! - if (x = 0) { - // error handling - } - else { - // code that only works for nonzero x - } +Assignment expressions using the walrus operator `:=` assign a variable in an +expression:: -The error is a simple typo: ``x = 0``, which assigns 0 to the variable ``x``, -was written while the comparison ``x == 0`` is certainly what was intended. + while chunk := fp.read(200): + print(chunk) -Many alternatives have been proposed. Most are hacks that save some typing but -use arbitrary or cryptic syntax or keywords, and fail the simple criterion for -language change proposals: it should intuitively suggest the proper meaning to a -human reader who has not yet been introduced to the construct. - -An interesting phenomenon is that most experienced Python programmers recognize -the ``while True`` idiom and don't seem to be missing the assignment in -expression construct much; it's only newcomers who express a strong desire to -add this to the language. - -There's an alternative way of spelling this that seems attractive but is -generally less robust than the "while True" solution:: - - line = f.readline() - while line: - ... # do something with line... - line = f.readline() - -The problem with this is that if you change your mind about exactly how you get -the next line (e.g. you want to change it into ``sys.stdin.readline()``) you -have to remember to change two places in your program -- the second occurrence -is hidden at the bottom of the loop. - -The best approach is to use iterators, making it possible to loop through -objects using the ``for`` statement. For example, :term:`file objects -` support the iterator protocol, so you can write simply:: - - for line in f: - ... # do something with line... +See :pep:`572` for more information. @@ -620,8 +571,7 @@ whether an instance or a class implements a particular ABC. The :class:`~collections.abc.MutableMapping`. For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. There is also a tool, -PyChecker, which can be used to find problems due to subclassing. +by an appropriate test discipline for components. A good test suite for a module can both provide a regression test and serve as a module interface specification and a set of examples. Many Python modules can @@ -649,7 +599,15 @@ sloppy and not write test cases at all. Why is there no goto? --------------------- -You can use exceptions to provide a "structured goto" that even works across +In the 1970s people realized that unrestricted goto could lead +to messy "spaghetti" code that was hard to understand and revise. +In a high-level language, it is also unneeded as long as there +are ways to branch (in Python, with ``if`` statements and ``or``, +``and``, and ``if-else`` expressions) and loop (with ``while`` +and ``for`` statements, possibly containing ``continue`` and ``break``). + +One can also use exceptions to provide a "structured goto" +that works even across function calls. Many feel that exceptions can conveniently emulate all reasonable uses of the "go" or "goto" constructs of C, Fortran, and other languages. For example:: diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2ad27658705258..aecb56eaa4fd2f 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -89,7 +89,7 @@ For bytes, :c:func:`PyBytes_Size` returns its length and length. Note that Python bytes objects may contain null bytes so C's :c:func:`strlen` should not be used. -To test the type of an object, first make sure it isn't *NULL*, and then use +To test the type of an object, first make sure it isn't ``NULL``, and then use :c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. There is also a high-level API to Python objects which is provided by the diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 3ef553e8acb43c..eee3c3c203efa5 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -17,12 +17,13 @@ What is Python? Python is an interpreted, interactive, object-oriented programming language. It incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. Python combines remarkable power with very clear syntax. -It has interfaces to many system calls and libraries, as well as to various -window systems, and is extensible in C or C++. It is also usable as an -extension language for applications that need a programmable interface. -Finally, Python is portable: it runs on many Unix variants, on the Mac, and on -Windows 2000 and later. +types, and classes. It supports multiple programming paradigms beyond +object-oriented programming, such as procedural and functional programming. +Python combines remarkable power with very clear syntax. It has interfaces to +many system calls and libraries, as well as to various window systems, and is +extensible in C or C++. It is also usable as an extension language for +applications that need a programmable interface. Finally, Python is portable: +it runs on many Unix variants including Linux and macOS, and on Windows. To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to Python `_ links to other @@ -295,8 +296,8 @@ How stable is Python? --------------------- Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. Currently there are -usually around 18 months between major releases. +months since 1991, and this seems likely to continue. As of version 3.9, +Python will have a major new release every 12 months (:pep:`602`). The developers issue "bugfix" releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third @@ -314,8 +315,8 @@ be maintained after January 1, 2020 ` How many people are using Python? --------------------------------- -There are probably tens of thousands of users, though it's difficult to obtain -an exact count. +There are probably millions of users, though it's difficult to obtain an exact +count. Python is available for free download, so there are no sales figures, and it's available from many different sites and packaged with many Linux distributions, diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index a00c6a053ef120..4caff0dd287d69 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -29,50 +29,36 @@ Python distribution (normally available as Tools/scripts/idle), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The -Pythonwin debugger colors breakpoints and has quite a few cool features such as -debugging non-Pythonwin programs. Pythonwin is available as part of the `Python -for Windows Extensions `__ project and -as a part of the ActivePython distribution (see -https://www.activestate.com/activepython\ ). - -`Boa Constructor `_ is an IDE and GUI -builder that uses wxWidgets. It offers visual frame creation and manipulation, -an object inspector, many views on the source like object browsers, inheritance -hierarchies, doc string generated html documentation, an advanced debugger, -integrated help, and Zope support. +PythonWin debugger colors breakpoints and has quite a few cool features such as +debugging non-PythonWin programs. PythonWin is available as part of +`pywin32 `_ project and +as a part of the +`ActivePython `_ distribution. `Eric `_ is an IDE built on PyQt and the Scintilla editing component. -Pydb is a version of the standard Python debugger pdb, modified for use with DDD -(Data Display Debugger), a popular graphical debugger front end. Pydb can be -found at http://bashdb.sourceforge.net/pydb/ and DDD can be found at -https://www.gnu.org/software/ddd. +`trepan3k `_ is a gdb-like debugger. + +`Visual Studio Code `_ is an IDE with debugging +tools that integrates with version-control software. There are a number of commercial Python IDEs that include graphical debuggers. They include: -* Wing IDE (https://wingware.com/) -* Komodo IDE (https://komodoide.com/) -* PyCharm (https://www.jetbrains.com/pycharm/) +* `Wing IDE `_ +* `Komodo IDE `_ +* `PyCharm `_ -Is there a tool to help find bugs or perform static analysis? +Are there tools to help find bugs or perform static analysis? ------------------------------------------------------------- Yes. -PyChecker is a static analysis tool that finds bugs in Python source code and -warns about code complexity and style. You can get PyChecker from -http://pychecker.sourceforge.net/. - -`Pylint `_ is another tool that checks -if a module satisfies a coding standard, and also makes it possible to write -plug-ins to add a custom feature. In addition to the bug checking that -PyChecker performs, Pylint offers some additional features such as checking line -length, whether variable names are well-formed according to your coding -standard, whether declared interfaces are fully implemented, and more. -https://docs.pylint.org/ provides a full list of Pylint's features. +`Pylint `_ and +`Pyflakes `_ do basic checking that will +help you catch bugs sooner. Static type checkers such as `Mypy `_, `Pyre `_, and @@ -104,11 +90,12 @@ then compiles the generated C code and links it with the rest of the Python interpreter to form a self-contained binary which acts exactly like your script. Obviously, freeze requires a C compiler. There are several other utilities -which don't. One is Thomas Heller's py2exe (Windows only) at - - http://www.py2exe.org/ +which don't: -Another tool is Anthony Tuininga's `cx_Freeze `_. +* `py2exe `_ for Windows binaries +* `py2app `_ for Mac OS X binaries +* `cx_Freeze `_ for cross-platform + binaries Are there coding standards or a style guide for Python programs? @@ -518,14 +505,14 @@ desired effect in a number of ways. 1) By returning a tuple of the results:: - def func2(a, b): - a = 'new-value' # a and b are local names - b = b + 1 # assigned to new objects - return a, b # return new values - - x, y = 'old-value', 99 - x, y = func2(x, y) - print(x, y) # output: new-value 100 + >>> def func1(a, b): + ... a = 'new-value' # a and b are local names + ... b = b + 1 # assigned to new objects + ... return a, b # return new values + ... + >>> x, y = 'old-value', 99 + >>> func1(x, y) + ('new-value', 100) This is almost always the clearest solution. @@ -533,38 +520,41 @@ desired effect in a number of ways. 3) By passing a mutable (changeable in-place) object:: - def func1(a): - a[0] = 'new-value' # 'a' references a mutable list - a[1] = a[1] + 1 # changes a shared object - - args = ['old-value', 99] - func1(args) - print(args[0], args[1]) # output: new-value 100 + >>> def func2(a): + ... a[0] = 'new-value' # 'a' references a mutable list + ... a[1] = a[1] + 1 # changes a shared object + ... + >>> args = ['old-value', 99] + >>> func2(args) + >>> args + ['new-value', 100] 4) By passing in a dictionary that gets mutated:: - def func3(args): - args['a'] = 'new-value' # args is a mutable dictionary - args['b'] = args['b'] + 1 # change it in-place - - args = {'a': 'old-value', 'b': 99} - func3(args) - print(args['a'], args['b']) + >>> def func3(args): + ... args['a'] = 'new-value' # args is a mutable dictionary + ... args['b'] = args['b'] + 1 # change it in-place + ... + >>> args = {'a': 'old-value', 'b': 99} + >>> func3(args) + >>> args + {'a': 'new-value', 'b': 100} 5) Or bundle up values in a class instance:: - class callByRef: - def __init__(self, /, **args): - for key, value in args.items(): - setattr(self, key, value) - - def func4(args): - args.a = 'new-value' # args is a mutable callByRef - args.b = args.b + 1 # change object in-place - - args = callByRef(a='old-value', b=99) - func4(args) - print(args.a, args.b) + >>> class Namespace: + ... def __init__(self, /, **args): + ... for key, value in args.items(): + ... setattr(self, key, value) + ... + >>> def func4(args): + ... args.a = 'new-value' # args is a mutable Namespace + ... args.b = args.b + 1 # change object in-place + ... + >>> args = Namespace(a='old-value', b=99) + >>> func4(args) + >>> vars(args) + {'a': 'new-value', 'b': 100} There's almost never a good reason to get this complicated. @@ -659,7 +649,7 @@ How can my code discover the name of an object? ----------------------------------------------- Generally speaking, it can't, because objects don't really have names. -Essentially, assignment always binds a name to a value; The same is true of +Essentially, assignment always binds a name to a value; the same is true of ``def`` and ``class`` statements, but in that case the value is a callable. Consider the following code:: @@ -779,30 +769,23 @@ A slash in the argument list of a function denotes that the parameters prior to it are positional-only. Positional-only parameters are the ones without an externally-usable name. Upon calling a function that accepts positional-only parameters, arguments are mapped to parameters based solely on their position. -For example, :func:`pow` is a function that accepts positional-only parameters. -Its documentation looks like this:: - - >>> help(pow) - Help on built-in function pow in module builtins: +For example, :func:`divmod` is a function that accepts positional-only +parameters. Its documentation looks like this:: - pow(x, y, z=None, /) - Equivalent to x**y (with two arguments) or x**y % z (with three arguments) + >>> help(divmod) + Help on built-in function divmod in module builtins: - Some types, such as ints, are able to use a more efficient algorithm when - invoked using the three argument form. + divmod(x, y, /) + Return the tuple (x//y, x%y). Invariant: div*y + mod == x. -The slash at the end of the parameter list means that all three parameters are -positional-only. Thus, calling :func:`pow` with keyword arguments would lead to -an error:: +The slash at the end of the parameter list means that both parameters are +positional-only. Thus, calling :func:`divmod` with keyword arguments would lead +to an error:: - >>> pow(x=3, y=4) + >>> divmod(x=3, y=4) Traceback (most recent call last): File "", line 1, in - TypeError: pow() takes no keyword arguments - -Note that as of this writing this is only documentational and no valid syntax -in Python, although there is :pep:`570`, which proposes a syntax for -position-only parameters in Python. + TypeError: divmod() takes no keyword arguments Numbers and strings @@ -1026,7 +1009,7 @@ That's a tough one, in general. First, here are a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ - focusses on :term:`CPython`. + focuses on :term:`CPython`. * Behaviour can vary across operating systems, especially when talking about I/O or multi-threading. * You should always find the hot spots in your program *before* attempting to @@ -1139,7 +1122,7 @@ trailing newline from a string. How do I iterate over a sequence in reverse order? -------------------------------------------------- -Use the :func:`reversed` built-in function, which is new in Python 2.4:: +Use the :func:`reversed` built-in function:: for x in reversed(sequence): ... # do something with x ... @@ -1147,11 +1130,6 @@ Use the :func:`reversed` built-in function, which is new in Python 2.4:: This won't touch your original sequence, but build a new copy with reversed order to iterate over. -With Python 2.3, you can use an extended slice syntax:: - - for x in sequence[::-1]: - ... # do something with x ... - How do you remove duplicates from a list? ----------------------------------------- @@ -1181,6 +1159,21 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. +How do you remove multiple items from a list +-------------------------------------------- + +As with removing duplicates, explicitly iterating in reverse with a +delete condition is one possibility. However, it is easier and faster +to use slice replacement with an implicit or explicit forward iteration. +Here are three variations.:: + + mylist[:] = filter(keep_function, mylist) + mylist[:] = (x for x in mylist if keep_condition) + mylist[:] = [x for x in mylist if keep_condition] + +The list comprehension may be fastest. + + How do you make an array in Python? ----------------------------------- @@ -1501,8 +1494,8 @@ to uppercase:: Here the ``UpperOut`` class redefines the ``write()`` method to convert the argument string to uppercase before calling the underlying -``self.__outfile.write()`` method. All other methods are delegated to the -underlying ``self.__outfile`` object. The delegation is accomplished via the +``self._outfile.write()`` method. All other methods are delegated to the +underlying ``self._outfile`` object. The delegation is accomplished via the ``__getattr__`` method; consult :ref:`the language reference ` for more information about controlling attribute access. diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index a181086e9ce627..186dac2e255b37 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -140,11 +140,9 @@ offender. How do I make an executable from a Python script? ------------------------------------------------- -See `cx_Freeze `_ for a distutils extension -that allows you to create console and GUI executables from Python code. -`py2exe `_, the most popular extension for building -Python 2.x-based executables, does not yet support Python 3 but a version that -does is in development. +See `cx_Freeze `_ and +`py2exe `_, both are distutils extensions +that allow you to create console and GUI executables from Python code. Is a ``*.pyd`` file the same as a DLL? @@ -279,7 +277,7 @@ in batch mode. How do I check for a keypress without blocking? ----------------------------------------------- -Use the msvcrt module. This is a standard Windows-specific extension module. +Use the :mod:`msvcrt` module. This is a standard Windows-specific extension module. It defines a function ``kbhit()`` which checks whether a keyboard hit is present, and ``getch()`` which gets one character without echoing it. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0f2a3a1fdf0510..d94fc4469c50af 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -189,6 +189,10 @@ Glossary A list of bytecode instructions can be found in the documentation for :ref:`the dis module `. + callback + A subroutine function which is passed as an argument to be executed at + some point in the future. + class A template for creating user-defined objects. Class definitions normally contain method definitions which operate on instances of the @@ -247,7 +251,7 @@ Glossary Fortran contiguous arrays, the first index varies the fastest. coroutine - Coroutines is a more generalized form of subroutines. Subroutines are + Coroutines are a more generalized form of subroutines. Subroutines are entered at one point and exited at another point. Coroutines can be entered, exited, and resumed at many different points. They can be implemented with the :keyword:`async def` statement. See also @@ -304,6 +308,12 @@ Glossary keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. Called a hash in Perl. + dictionary comprehension + A compact way to process all or part of the elements in an iterable and + return a dictionary with the results. ``results = {n: n ** 2 for n in + range(10)}`` generates a dictionary containing key ``n`` mapped to + value ``n ** 2``. See :ref:`comprehensions`. + dictionary view The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and :meth:`dict.items` are called dictionary views. They provide a dynamic @@ -583,7 +593,7 @@ Glossary and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define with an :meth:`__iter__` method or with a :meth:`__getitem__` method - that implements :term:`Sequence` semantics. + that implements :term:`Sequence ` semantics. Iterables can be used in a :keyword:`for` loop and in many other places where a sequence is @@ -739,17 +749,28 @@ Glossary also :term:`immutable`. named tuple - Any tuple-like class whose indexable elements are also accessible using - named attributes (for example, :func:`time.localtime` returns a - tuple-like object where the *year* is accessible either with an - index such as ``t[0]`` or with a named attribute like ``t.tm_year``). - - A named tuple can be a built-in type such as :class:`time.struct_time`, - or it can be created with a regular class definition. A full featured - named tuple can also be created with the factory function - :func:`collections.namedtuple`. The latter approach automatically - provides extra features such as a self-documenting representation like - ``Employee(name='jones', title='programmer')``. + The term "named tuple" applies to any type or class that inherits from + tuple and whose indexable elements are also accessible using named + attributes. The type or class may have other features as well. + + Several built-in types are named tuples, including the values returned + by :func:`time.localtime` and :func:`os.stat`. Another example is + :data:`sys.float_info`:: + + >>> sys.float_info[1] # indexed access + 1024 + >>> sys.float_info.max_exp # named field access + 1024 + >>> isinstance(sys.float_info, tuple) # kind of tuple + True + + Some named tuples are built-in types (such as the above examples). + Alternatively, a named tuple can be created from a regular class + definition that inherits from :class:`tuple` and that defines named + fields. Such a class can be written by hand or it can be created with + the factory function :func:`collections.namedtuple`. The latter + technique also adds some extra methods that may not be found in + hand-written or built-in named tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -813,9 +834,11 @@ Glossary .. _positional-only_parameter: * :dfn:`positional-only`: specifies an argument that can be supplied only - by position. Python has no syntax for defining positional-only - parameters. However, some built-in functions have positional-only - parameters (e.g. :func:`abs`). + by position. Positional-only parameters can be defined by including a + ``/`` character in the parameter list of the function definition after + them, for example *posonly1* and *posonly2* in the following:: + + def func(posonly1, posonly2, /, positional_or_keyword): ... .. _keyword-only_parameter: @@ -1007,7 +1030,13 @@ Glossary :meth:`index`, :meth:`__contains__`, and :meth:`__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.register`. + :func:`~abc.ABCMeta.register`. + + set comprehension + A compact way to process all or part of the elements in an iterable and + return a set with the results. ``results = {c for c in 'abracadabra' if + c not in 'abc'}`` generates the set of strings ``{'r', 'd'}``. See + :ref:`comprehensions`. single dispatch A form of :term:`generic function` dispatch where the implementation is @@ -1032,14 +1061,6 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. - struct sequence - A tuple with named elements. Struct sequences expose an interface similar - to :term:`named tuple` in that elements can be accessed either by - index or as an attribute. However, they do not have any of the named tuple - methods like :meth:`~collections.somenamedtuple._make` or - :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences - include :data:`sys.float_info` and the return value of :func:`os.stat`. - text encoding A codec which encodes Unicode strings to bytes. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index cfd9f2e4075c4a..50041829b8c388 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1070,7 +1070,7 @@ Currently Argument Clinic supports only a few return converters: DecodeFSDefault None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``const char *``; return a NULL +error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL`` pointer to indicate an error. (There's also an experimental ``NoneType`` converter, which lets you diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index b29e590b20cba8..4d25d2ccb66507 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -117,7 +117,7 @@ The important points to remember are: * non-data descriptors may be overridden by instance dictionaries. The object returned by ``super()`` also has a custom :meth:`__getattribute__` -method for invoking descriptors. The call ``super(B, obj).m()`` searches +method for invoking descriptors. The attribute lookup ``super(B, obj).m`` searches ``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, ``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a @@ -312,14 +312,12 @@ Running the interpreter shows how the function descriptor works in practice:: >>> d.f > - # Internally, the bound method stores the underlying function, - # the bound instance, and the class of the bound instance. + # Internally, the bound method stores the underlying function and + # the bound instance. >>> d.f.__func__ >>> d.f.__self__ <__main__.D object at 0x1012e1f98> - >>> d.f.__class__ - Static Methods and Class Methods diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index f8f2aac70f9b06..74e861480d2ff8 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1229,9 +1229,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__, Python documentation diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 909deb5fed33ff..f0081e4ec28905 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -272,9 +272,7 @@ should instead read: Available static markers ------------------------ -.. I'm reusing the "c:function" type for markers - -.. c:function:: function__entry(str filename, str funcname, int lineno) +.. object:: function__entry(str filename, str funcname, int lineno) This marker indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. @@ -290,7 +288,7 @@ Available static markers * ``$arg3`` : ``int`` line number -.. c:function:: function__return(str filename, str funcname, int lineno) +.. object:: function__return(str filename, str funcname, int lineno) This marker is the converse of :c:func:`function__entry`, and indicates that execution of a Python function has ended (either via ``return``, or via an @@ -298,7 +296,7 @@ Available static markers The arguments are the same as for :c:func:`function__entry` -.. c:function:: line(str filename, str funcname, int lineno) +.. object:: line(str filename, str funcname, int lineno) This marker indicates a Python line is about to be executed. It is the equivalent of line-by-line tracing with a Python profiler. It is @@ -306,24 +304,24 @@ Available static markers The arguments are the same as for :c:func:`function__entry`. -.. c:function:: gc__start(int generation) +.. object:: gc__start(int generation) Fires when the Python interpreter starts a garbage collection cycle. ``arg0`` is the generation to scan, like :func:`gc.collect()`. -.. c:function:: gc__done(long collected) +.. object:: gc__done(long collected) Fires when the Python interpreter finishes a garbage collection cycle. ``arg0`` is the number of collected objects. -.. c:function:: import__find__load__start(str modulename) +.. object:: import__find__load__start(str modulename) Fires before :mod:`importlib` attempts to find and load the module. ``arg0`` is the module name. .. versionadded:: 3.7 -.. c:function:: import__find__load__done(str modulename, int found) +.. object:: import__find__load__done(str modulename, int found) Fires after :mod:`importlib`'s find_and_load function is called. ``arg0`` is the module name, ``arg1`` indicates if module was @@ -332,7 +330,7 @@ Available static markers .. versionadded:: 3.7 -.. c:function:: audit(str event, void *tuple) +.. object:: audit(str event, void *tuple) Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` @@ -375,14 +373,14 @@ If this file is installed in SystemTap's tapset directory (e.g. ``/usr/share/systemtap/tapset``), then these additional probepoints become available: -.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.entry(str filename, str funcname, int lineno, frameptr) This probe point indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. -.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.return(str filename, str funcname, int lineno, frameptr) - This probe point is the converse of :c:func:`python.function.return`, and + This probe point is the converse of ``python.function.return``, and indicates that execution of a Python function has ended (either via ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 71f9fc920fdfc9..5777a4c5031f85 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -948,6 +948,41 @@ This variant shows how you can e.g. apply configuration for particular loggers machinery in the main process (even though the logging events are generated in the worker processes) to direct the messages to the appropriate destinations. +Using concurrent.futures.ProcessPoolExecutor +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to use :class:`concurrent.futures.ProcessPoolExecutor` to start +your worker processes, you need to create the queue slightly differently. +Instead of + +.. code-block:: python + + queue = multiprocessing.Queue(-1) + +you should use + +.. code-block:: python + + queue = multiprocessing.Manager().Queue(-1) # also works with the examples above + +and you can then replace the worker creation from this:: + + workers = [] + for i in range(10): + worker = multiprocessing.Process(target=worker_process, + args=(queue, worker_configurer)) + workers.append(worker) + worker.start() + for w in workers: + w.join() + +to this (remembering to first import :mod:`concurrent.futures`):: + + with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: + for i in range(10): + executor.submit(worker_process, queue, worker_configurer) + + Using file rotation ------------------- @@ -1153,7 +1188,7 @@ to the above, as in the following example:: class StyleAdapter(logging.LoggerAdapter): def __init__(self, logger, extra=None): - super(StyleAdapter, self).__init__(logger, extra or {}) + super().__init__(logger, extra or {}) def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): @@ -1333,7 +1368,7 @@ An example dictionary-based configuration ----------------------------------------- Below is an example of a logging configuration dictionary - it's taken from -the `documentation on the Django project `_. +the `documentation on the Django project `_. This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: LOGGING = { @@ -1389,7 +1424,7 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration } For more information about this configuration, you can see the `relevant -section `_ +section `_ of the Django documentation. .. _cookbook-rotator-namer: @@ -1748,7 +1783,7 @@ as in the following complete example:: return tuple(o) elif isinstance(o, unicode): return o.encode('unicode_escape').decode('ascii') - return super(Encoder, self).default(o) + return super().default(o) class StructuredMessage: def __init__(self, message, /, **kwargs): @@ -2140,11 +2175,11 @@ class, as shown in the following example:: """ Format an exception so that it prints on a single line. """ - result = super(OneLineExceptionFormatter, self).formatException(exc_info) + result = super().formatException(exc_info) return repr(result) # or format into one line however you want to def format(self, record): - s = super(OneLineExceptionFormatter, self).format(record) + s = super().format(record) if record.exc_text: s = s.replace('\n', '') + '|' return s @@ -2266,9 +2301,9 @@ The script just arranges to decorate ``foo`` with a decorator which will do the conditional logging that's required. The decorator takes a logger as a parameter and attaches a memory handler for the duration of the call to the decorated function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. +a level at which flushing should occur, and a capacity for the buffer (number of +records buffered). These default to a :class:`~logging.StreamHandler` which +writes to ``sys.stderr``, ``logging.ERROR`` and ``100`` respectively. Here's the script:: @@ -2709,3 +2744,238 @@ And if we want less: In this case, the commands don't print anything to the console, since nothing at ``WARNING`` level or above is logged by them. + +.. _qt-gui: + +A Qt GUI for logging +-------------------- + +A question that comes up from time to time is about how to log to a GUI +application. The `Qt `_ framework is a popular +cross-platform UI framework with Python bindings using `PySide2 +`_ or `PyQt5 +`_ libraries. + +The following example shows how to log to a Qt GUI. This introduces a simple +``QtHandler`` class which takes a callable, which should be a slot in the main +thread that does GUI updates. A worker thread is also created to show how you +can log to the GUI from both the UI itself (via a button for manual logging) +as well as a worker thread doing work in the background (here, just logging +messages at random levels with random short delays in between). + +The worker thread is implemented using Qt's ``QThread`` class rather than the +:mod:`threading` module, as there are circumstances where one has to use +``QThread``, which offers better integration with other ``Qt`` components. + +The code should work with recent releases of either ``PySide2`` or ``PyQt5``. +You should be able to adapt the approach to earlier versions of Qt. Please +refer to the comments in the code snippet for more detailed information. + +.. code-block:: python3 + + import datetime + import logging + import random + import sys + import time + + # Deal with minor differences between PySide2 and PyQt5 + try: + from PySide2 import QtCore, QtGui, QtWidgets + Signal = QtCore.Signal + Slot = QtCore.Slot + except ImportError: + from PyQt5 import QtCore, QtGui, QtWidgets + Signal = QtCore.pyqtSignal + Slot = QtCore.pyqtSlot + + + logger = logging.getLogger(__name__) + + + # + # Signals need to be contained in a QObject or subclass in order to be correctly + # initialized. + # + class Signaller(QtCore.QObject): + signal = Signal(str, logging.LogRecord) + + # + # Output to a Qt GUI is only supposed to happen on the main thread. So, this + # handler is designed to take a slot function which is set up to run in the main + # thread. In this example, the function takes a string argument which is a + # formatted log message, and the log record which generated it. The formatted + # string is just a convenience - you could format a string for output any way + # you like in the slot function itself. + # + # You specify the slot function to do whatever GUI updates you want. The handler + # doesn't know or care about specific UI elements. + # + class QtHandler(logging.Handler): + def __init__(self, slotfunc, *args, **kwargs): + super().__init__(*args, **kwargs) + self.signaller = Signaller() + self.signaller.signal.connect(slotfunc) + + def emit(self, record): + s = self.format(record) + self.signaller.signal.emit(s, record) + + # + # This example uses QThreads, which means that the threads at the Python level + # are named something like "Dummy-1". The function below gets the Qt name of the + # current thread. + # + def ctname(): + return QtCore.QThread.currentThread().objectName() + + + # + # Used to generate random levels for logging. + # + LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, + logging.CRITICAL) + + # + # This worker class represents work that is done in a thread separate to the + # main thread. The way the thread is kicked off to do work is via a button press + # that connects to a slot in the worker. + # + # Because the default threadName value in the LogRecord isn't much use, we add + # a qThreadName which contains the QThread name as computed above, and pass that + # value in an "extra" dictionary which is used to update the LogRecord with the + # QThread name. + # + # This example worker just outputs messages sequentially, interspersed with + # random delays of the order of a few seconds. + # + class Worker(QtCore.QObject): + @Slot() + def start(self): + extra = {'qThreadName': ctname() } + logger.debug('Started work', extra=extra) + i = 1 + # Let the thread run until interrupted. This allows reasonably clean + # thread termination. + while not QtCore.QThread.currentThread().isInterruptionRequested(): + delay = 0.5 + random.random() * 2 + time.sleep(delay) + level = random.choice(LEVELS) + logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) + i += 1 + + # + # Implement a simple UI for this cookbook example. This contains: + # + # * A read-only text edit window which holds formatted log messages + # * A button to start work and log stuff in a separate thread + # * A button to log something from the main thread + # * A button to clear the log window + # + class Window(QtWidgets.QWidget): + + COLORS = { + logging.DEBUG: 'black', + logging.INFO: 'blue', + logging.WARNING: 'orange', + logging.ERROR: 'red', + logging.CRITICAL: 'purple', + } + + def __init__(self, app): + super().__init__() + self.app = app + self.textedit = te = QtWidgets.QPlainTextEdit(self) + # Set whatever the default monospace font is for the platform + f = QtGui.QFont('nosuchfont') + f.setStyleHint(f.Monospace) + te.setFont(f) + te.setReadOnly(True) + PB = QtWidgets.QPushButton + self.work_button = PB('Start background work', self) + self.log_button = PB('Log a message at a random level', self) + self.clear_button = PB('Clear log window', self) + self.handler = h = QtHandler(self.update_status) + # Remember to use qThreadName rather than threadName in the format string. + fs = '%(asctime)s %(qThreadName)-12s %(levelname)-8s %(message)s' + formatter = logging.Formatter(fs) + h.setFormatter(formatter) + logger.addHandler(h) + # Set up to terminate the QThread when we exit + app.aboutToQuit.connect(self.force_quit) + + # Lay out all the widgets + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(te) + layout.addWidget(self.work_button) + layout.addWidget(self.log_button) + layout.addWidget(self.clear_button) + self.setFixedSize(900, 400) + + # Connect the non-worker slots and signals + self.log_button.clicked.connect(self.manual_update) + self.clear_button.clicked.connect(self.clear_display) + + # Start a new worker thread and connect the slots for the worker + self.start_thread() + self.work_button.clicked.connect(self.worker.start) + # Once started, the button should be disabled + self.work_button.clicked.connect(lambda : self.work_button.setEnabled(False)) + + def start_thread(self): + self.worker = Worker() + self.worker_thread = QtCore.QThread() + self.worker.setObjectName('Worker') + self.worker_thread.setObjectName('WorkerThread') # for qThreadName + self.worker.moveToThread(self.worker_thread) + # This will start an event loop in the worker thread + self.worker_thread.start() + + def kill_thread(self): + # Just tell the worker to stop, then tell it to quit and wait for that + # to happen + self.worker_thread.requestInterruption() + if self.worker_thread.isRunning(): + self.worker_thread.quit() + self.worker_thread.wait() + else: + print('worker has already exited.') + + def force_quit(self): + # For use when the window is closed + if self.worker_thread.isRunning(): + self.kill_thread() + + # The functions below update the UI and run in the main thread because + # that's where the slots are set up + + @Slot(str, logging.LogRecord) + def update_status(self, status, record): + color = self.COLORS.get(record.levelno, 'black') + s = '
%s
' % (color, status) + self.textedit.appendHtml(s) + + @Slot() + def manual_update(self): + # This function uses the formatted message passed in, but also uses + # information from the record to format the message in an appropriate + # color according to its severity (level). + level = random.choice(LEVELS) + extra = {'qThreadName': ctname() } + logger.log(level, 'Manually logged!', extra=extra) + + @Slot() + def clear_display(self): + self.textedit.clear() + + + def main(): + QtCore.QThread.currentThread().setObjectName('MainThread') + logging.getLogger().setLevel(logging.DEBUG) + app = QtWidgets.QApplication(sys.argv) + example = Window(app) + example.show() + sys.exit(app.exec_()) + + if __name__=='__main__': + main() diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 3be6bb380d663b..8608162bace357 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -119,7 +119,7 @@ Once you have your code well-tested you are ready to begin porting your code to Python 3! But to fully understand how your code is going to change and what you want to look out for while you code, you will want to learn what changes Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the `"What's New"`_ doc for each release of Python 3 and the +is reading the :ref:`"What's New" ` doc for each release of Python 3 and the `Porting to Python 3`_ book (which is free online). There is also a handy `cheat sheet`_ from the Python-Future project. @@ -302,10 +302,10 @@ If for some reason that doesn't work then you should make the version check be against Python 2 and not Python 3. To help explain this, let's look at an example. -Let's pretend that you need access to a feature of importlib_ that +Let's pretend that you need access to a feature of :mod:`importlib` that is available in Python's standard library since Python 3.3 and available for Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the ``importlib.abc`` module by doing the following:: +access e.g. the :mod:`importlib.abc` module by doing the following:: import sys @@ -426,12 +426,10 @@ can also explicitly state whether your APIs use textual or binary data, helping to make sure everything functions as expected in both versions of Python. -.. _2to3: https://docs.python.org/3/library/2to3.html .. _caniusepython3: https://pypi.org/project/caniusepython3 .. _cheat sheet: http://python-future.org/compatible_idioms.html .. _coverage.py: https://pypi.org/project/coverage .. _Futurize: http://python-future.org/automatic_conversion.html -.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib .. _importlib2: https://pypi.org/project/importlib2 .. _Modernize: https://python-modernize.readthedocs.io/ .. _mypy: http://mypy-lang.org/ @@ -447,6 +445,4 @@ to make sure everything functions as expected in both versions of Python. .. _tox: https://pypi.org/project/tox .. _trove classifier: https://pypi.org/classifiers -.. _"What's New": https://docs.python.org/3/whatsnew/index.html - .. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index bc71d85a83e921..b5c2152ec7004d 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -319,7 +319,7 @@ inside-out. In Python, you use ``socket.setblocking(0)`` to make it non-blocking. In C, it's more complex, (for one thing, you'll need to choose between the BSD flavor -``O_NONBLOCK`` and the almost indistinguishable Posix flavor ``O_NDELAY``, which +``O_NONBLOCK`` and the almost indistinguishable POSIX flavor ``O_NDELAY``, which is completely different from ``TCP_NODELAY``), but it's the exact same idea. You do this after creating the socket, but before using it. (Actually, if you're nuts, you can switch back and forth.) diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 24c3235e4add94..51bd64bfc232ca 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -57,14 +57,14 @@ their corresponding code points: ... 007B '{'; LEFT CURLY BRACKET ... - 2167 'Ⅶ': ROMAN NUMERAL EIGHT - 2168 'Ⅸ': ROMAN NUMERAL NINE + 2167 'Ⅷ'; ROMAN NUMERAL EIGHT + 2168 'Ⅸ'; ROMAN NUMERAL NINE ... - 265E '♞': BLACK CHESS KNIGHT - 265F '♟': BLACK CHESS PAWN + 265E '♞'; BLACK CHESS KNIGHT + 265F '♟'; BLACK CHESS PAWN ... - 1F600 '😀': GRINNING FACE - 1F609 '😉': WINKING FACE + 1F600 '😀'; GRINNING FACE + 1F609 '😉'; WINKING FACE ... Strictly, these definitions imply that it's meaningless to say 'this is diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 046a88af62f0b3..12d525771ddc28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -97,7 +97,7 @@ schemes. For example, you can make an FTP request like so:: In the case of HTTP, there are two extra things that Request objects allow you to do: First, you can pass data to be sent to the server. Second, you can pass -extra information ("metadata") *about* the data or the about request itself, to +extra information ("metadata") *about* the data or about the request itself, to the server - this information is sent as HTTP "headers". Let's look at each of these in turn. diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c index 13d16f5424ae4d..f361baf830dd1b 100644 --- a/Doc/includes/custom.c +++ b/Doc/includes/custom.c @@ -35,6 +35,11 @@ PyInit_custom(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c index 6477a19dafed75..5bacab7a2a9714 100644 --- a/Doc/includes/custom2.c +++ b/Doc/includes/custom2.c @@ -128,6 +128,11 @@ PyInit_custom2(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c index 213d0864ce1ca8..2b7a99ecf96c76 100644 --- a/Doc/includes/custom3.c +++ b/Doc/includes/custom3.c @@ -179,6 +179,11 @@ PyInit_custom3(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c index b0b2906dbdc863..584992fc5f1a8a 100644 --- a/Doc/includes/custom4.c +++ b/Doc/includes/custom4.c @@ -193,6 +193,11 @@ PyInit_custom4(void) return NULL; Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { + Py_DECREF(&CustomType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 0dcfbfb4025c85..2fc1570e654db6 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -41,7 +41,7 @@ def main(): directory = '.' # Create the message msg = EmailMessage() - msg['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) + msg['Subject'] = f'Contents of directory {os.path.abspath(directory)}' msg['To'] = ', '.join(args.recipients) msg['From'] = args.sender msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' diff --git a/Doc/includes/email-mime.py b/Doc/includes/email-mime.py index c610242f11f843..6af2be0b08a48d 100644 --- a/Doc/includes/email-mime.py +++ b/Doc/includes/email-mime.py @@ -14,7 +14,7 @@ # family = the list of all recipients' email addresses msg['From'] = me msg['To'] = ', '.join(family) -msg.preamble = 'Our family reunion' +msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' # Open the files in binary mode. Use imghdr to figure out the # MIME subtype for each specific image. diff --git a/Doc/includes/email-simple.py b/Doc/includes/email-simple.py index f69ef40ff04c93..07dc30fd066eac 100644 --- a/Doc/includes/email-simple.py +++ b/Doc/includes/email-simple.py @@ -12,7 +12,7 @@ # me == the sender's email address # you == the recipient's email address -msg['Subject'] = 'The contents of %s' % textfile +msg['Subject'] = f'The contents of {textfile}' msg['From'] = me msg['To'] = you diff --git a/Doc/includes/email-unpack.py b/Doc/includes/email-unpack.py index e0a7f01f58bb59..c8cb0be4560830 100644 --- a/Doc/includes/email-unpack.py +++ b/Doc/includes/email-unpack.py @@ -43,7 +43,7 @@ def main(): if not ext: # Use a generic bag-of-bits extension ext = '.bin' - filename = 'part-%03d%s' % (counter, ext) + filename = f'part-{counter:03d}{ext}' counter += 1 with open(os.path.join(args.directory, filename), 'wb') as fp: fp.write(part.get_payload(decode=True)) diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c index 76ff93948cfd67..b2c26e73ebaf7e 100644 --- a/Doc/includes/sublist.c +++ b/Doc/includes/sublist.c @@ -59,6 +59,11 @@ PyInit_sublist(void) return NULL; Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { + Py_DECREF(&SubListType); + Py_DECREF(m); + return NULL; + } + return m; } diff --git a/Doc/install/index.rst b/Doc/install/index.rst index a91606c0f38e61..ae0d0294ef7773 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -532,7 +532,7 @@ scripts will wind up in :file:`/usr/local/python/bin`. If you want them in python setup.py install --install-scripts=/usr/local/bin -(This performs an installation using the "prefix scheme," where the prefix is +(This performs an installation using the "prefix scheme", where the prefix is whatever your Python interpreter was installed with--- :file:`/usr/local/python` in this case.) @@ -1064,8 +1064,7 @@ normal libraries do. .. [#] This also means you could replace all existing COFF-libraries with OMF-libraries of the same name. -.. [#] Check https://www.sourceware.org/cygwin/ and http://www.mingw.org/ for more - information +.. [#] Check https://www.sourceware.org/cygwin/ for more information .. [#] Then you have no POSIX emulation available, but you also don't need :file:`cygwin1.dll`. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index dc44aa64b8e045..31e9b0bde07244 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -53,7 +53,7 @@ Key terms evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on both `GitHub `__ and - `BitBucket `__. + `Bitbucket `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index fa4b0a9645531c..c3ff3e607e7978 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -456,7 +456,7 @@ and off individually. They are described here in more detail. ------------------------------- .. module:: lib2to3 - :synopsis: the 2to3 library + :synopsis: The 2to3 library .. moduleauthor:: Guido van Rossum .. moduleauthor:: Collin Winter diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index e3d749e6017847..41399942d30308 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,7 +90,7 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | 4.0 | :pep:`563`: | +| annotations | 3.7.0b1 | 3.10 | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 26568dcbf8f532..bd653ab32bb9c4 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -106,7 +106,7 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. .. versionadded:: 3.8 diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index 7328907730fb10..2e917cf7321b85 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -208,6 +208,7 @@ number of frames must be filled in. .. method:: aifc.tell() + :noindex: Return the current write position in the output file. Useful in combination with :meth:`setmark`. @@ -232,6 +233,7 @@ number of frames must be filled in. .. method:: aifc.close() + :noindex: Close the AIFF file. The header of the file is updated to reflect the actual size of the audio data. After calling this method, the object can no longer be diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index b77a38ccd48577..63648ed81d8c1c 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -182,6 +182,10 @@ ArgumentParser objects .. versionchanged:: 3.5 *allow_abbrev* parameter was added. + .. versionchanged:: 3.8 + In previous versions, *allow_abbrev* also disabled grouping of short + flags such as ``-vv`` to mean ``-v -v``. + The following sections describe how each of these are used. @@ -778,10 +782,12 @@ how the command-line arguments should be handled. The supplied actions are: example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--verbose', '-v', action='count') + >>> parser.add_argument('--verbose', '-v', action='count', default=0) >>> parser.parse_args(['-vvv']) Namespace(verbose=3) + Note, the *default* will be ``None`` unless explicitly set to *0*. + * ``'help'`` - This prints a complete help message for all the options in the current parser and then exits. By default a help action is automatically added to the parser. See :class:`ArgumentParser` for details of how the @@ -806,6 +812,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) Namespace(foo=['f1', 'f2', 'f3', 'f4']) + .. versionadded:: 3.8 + You may also specify an arbitrary action by passing an Action subclass or other object that implements the same interface. The recommended way to do this is to extend :class:`Action`, overriding the ``__call__`` method @@ -817,7 +825,7 @@ An example of a custom action:: ... def __init__(self, option_strings, dest, nargs=None, **kwargs): ... if nargs is not None: ... raise ValueError("nargs not allowed") - ... super(FooAction, self).__init__(option_strings, dest, **kwargs) + ... super().__init__(option_strings, dest, **kwargs) ... def __call__(self, parser, namespace, values, option_string=None): ... print('%r %r %r' % (namespace, values, option_string)) ... setattr(namespace, self.dest, values) @@ -1098,9 +1106,8 @@ container should match the type_ specified:: usage: doors.py [-h] {1,2,3} doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) -Any object that supports the ``in`` operator can be passed as the *choices* -value, so :class:`dict` objects, :class:`set` objects, custom containers, -etc. are all supported. +Any container can be passed as the *choices* value, so :class:`list` objects, +:class:`set` objects, and custom containers are all supported. required @@ -1116,8 +1123,8 @@ keyword argument to :meth:`~ArgumentParser.add_argument`:: >>> parser.parse_args(['--foo', 'BAR']) Namespace(foo='BAR') >>> parser.parse_args([]) - usage: argparse.py [-h] [--foo FOO] - argparse.py: error: option --foo is required + usage: [-h] --foo FOO + : error: the following arguments are required: --foo As the example shows, if an option is marked as ``required``, :meth:`~ArgumentParser.parse_args` will report an error if that option is not @@ -1592,7 +1599,7 @@ Sub-commands stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default - ``False``. + ``False`` (added in 3.7) * help_ - help for sub-parser group in help output, by default ``None`` @@ -1748,6 +1755,9 @@ Sub-commands >>> parser.parse_args(['2', 'frobble']) Namespace(subparser_name='2', y='frobble') + .. versionchanged:: 3.7 + New *required* keyword argument. + FileType objects ^^^^^^^^^^^^^^^^ @@ -1998,7 +2008,14 @@ Exiting methods .. method:: ArgumentParser.exit(status=0, message=None) This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. + and, if given, it prints a *message* before that. The user can override + this method to handle these steps differently:: + + class ErrorCatchingArgumentParser(argparse.ArgumentParser): + def exit(self, status=0, message=None): + if status: + raise Exception(f'Exiting because of an error: {message}') + exit(status) .. method:: ArgumentParser.error(message) diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 1f95dd61b9fcd7..6658cca3bb5146 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -36,9 +36,9 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'L'`` | unsigned long | int | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | \(2) | +| ``'q'`` | signed long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | \(2) | +| ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -57,13 +57,6 @@ Notes: .. deprecated-removed:: 3.3 4.0 -(2) - The ``'q'`` and ``'Q'`` type codes are available only if - the platform C compiler used to build Python supports C :c:type:`long long`, - or, on Windows, :c:type:`__int64`. - - .. versionadded:: 3.3 - The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed through the :attr:`itemsize` attribute. @@ -83,7 +76,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. - .. audit-event:: array.__new__ "typecode initializer" + .. audit-event:: array.__new__ typecode,initializer array.array .. data:: typecodes @@ -166,8 +159,7 @@ The following data items and methods are also supported: Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, :exc:`EOFError` is raised, but the items that were available are still - inserted into the array. *f* must be a real built-in file object; something - else with a :meth:`read` method won't do. + inserted into the array. .. method:: array.fromlist(list) @@ -180,6 +172,8 @@ The following data items and methods are also supported: Deprecated alias for :meth:`frombytes`. + .. deprecated-removed:: 3.2 3.9 + .. method:: array.fromunicode(s) @@ -242,6 +236,8 @@ The following data items and methods are also supported: Deprecated alias for :meth:`tobytes`. + .. deprecated-removed:: 3.2 3.9 + .. method:: array.tounicode() diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 1884bea80e8047..67c6392ad7d2c4 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -101,12 +101,16 @@ Node classes node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), lineno=0, col_offset=0) +.. versionchanged:: 3.8 + + Class :class:`ast.Constant` is now used for all constants. + .. deprecated:: 3.8 - Class :class:`ast.Constant` is now used for all constants. Old classes - :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, + Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. + but they will be removed in future Python releases. In the meanwhile, + instantiating them will return an instance of a different class. .. _abstract-grammar: @@ -126,7 +130,7 @@ The abstract grammar is currently defined as follows: Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. @@ -145,11 +149,12 @@ and classes for traversing abstract syntax trees: modified to correspond to :pep:`484` "signature type comments", e.g. ``(str, int) -> List[str]``. - Also, setting ``feature_version`` to the minor version of an - earlier Python 3 version will attempt to parse using that version's - grammar. For example, setting ``feature_version=4`` will allow - the use of ``async`` and ``await`` as variable names. The lowest - supported value is 4; the highest is ``sys.version_info[1]``. + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. .. warning:: It is possible to crash the Python interpreter with a @@ -274,6 +279,13 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + .. class:: NodeTransformer() @@ -292,11 +304,11 @@ and classes for traversing abstract syntax trees: class RewriteName(NodeTransformer): def visit_Name(self, node): - return copy_location(Subscript( + return Subscript( value=Name(id='data', ctx=Load()), slice=Index(value=Constant(value=node.id)), ctx=node.ctx - ), node) + ) Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:`generic_visit` @@ -306,6 +318,14 @@ and classes for traversing abstract syntax trees: statement nodes), the visitor may also return a list of nodes rather than just a single node. + If :class:`NodeTransformer` introduces new nodes (that weren't part of + original tree) without giving them location information (such as + :attr:`lineno`), :func:`fix_missing_locations` should be called with + the new sub-tree to recalculate the location information:: + + tree = ast.parse('foo', mode='eval') + new_tree = fix_missing_locations(RewriteName().visit(tree)) + Usually you use the transformer like this:: node = YourTransformer().visit(node) @@ -314,13 +334,33 @@ and classes for traversing abstract syntax trees: .. function:: dump(node, annotate_fields=True, include_attributes=False) Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to ``False``. Attributes such as line + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to ``True``. + *include_attributes* can be set to true. .. seealso:: - `Green Tree Snakes `_, an external documentation resource, has good - details on working with Python ASTs. + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. \ No newline at end of file diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b7288036192979..d7ed96282fee18 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -73,7 +73,7 @@ event loop, no other Tasks can run in the same thread. When a Task executes an ``await`` expression, the running Task gets suspended, and the event loop executes the next Task. -To schedule a callback from a different OS thread, the +To schedule a :term:`callback` from another OS thread, the :meth:`loop.call_soon_threadsafe` method should be used. Example:: loop.call_soon_threadsafe(callback, *args) @@ -107,6 +107,16 @@ The :meth:`loop.run_in_executor` method can be used with a blocking code in a different OS thread without blocking the OS thread that the event loop runs in. +There is currently no way to schedule coroutines or callbacks directly +from a different process (such as one started with +:mod:`multiprocessing`). The :ref:`Event Loop Methods ` +section lists APIs that can read from pipes and watch file descriptors +without blocking the event loop. In addition, asyncio's +:ref:`Subprocess ` APIs provide a way to start a +process and communicate with it from the event loop. Lastly, the +aforementioned :meth:`loop.run_in_executor` method can also be used +with a :class:`concurrent.futures.ProcessPoolExecutor` to execute +code in a different process. .. _asyncio-handle-blocking: @@ -119,7 +129,7 @@ all concurrent asyncio Tasks and IO operations would be delayed by 1 second. An executor can be used to run a task in a different thread or even in -a different process to avoid blocking block the OS thread with the +a different process to avoid blocking the OS thread with the event loop. See the :meth:`loop.run_in_executor` method for more details. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8673f84e96382e..dde57e295aa688 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -5,6 +5,10 @@ Event Loop ========== +**Source code:** :source:`Lib/asyncio/events.py`, +:source:`Lib/asyncio/base_events.py` + +------------------------------------ .. rubric:: Preface @@ -34,8 +38,10 @@ an event loop: .. function:: get_event_loop() - Get the current event loop. If there is no current event loop set - in the current OS thread and :func:`set_event_loop` has not yet + Get the current event loop. + + If there is no current event loop set in the current OS thread, + the OS thread is main, and :func:`set_event_loop` has not yet been called, asyncio will create a new event loop and set it as the current one. @@ -173,8 +179,8 @@ Scheduling callbacks .. method:: loop.call_soon(callback, *args, context=None) - Schedule a *callback* to be called with *args* arguments at - the next iteration of the event loop. + Schedule the *callback* :term:`callback` to be called with + *args* arguments at the next iteration of the event loop. Callbacks are called in the order in which they are registered. Each callback will be called exactly once. @@ -303,7 +309,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, \*, name=None) +.. method:: loop.create_task(coro, *, name=None) Schedule the execution of a :ref:`coroutine`. Return a :class:`Task` object. @@ -338,10 +344,11 @@ Opening network connections ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_connection(protocol_factory, \ - host=None, port=None, \*, ssl=None, \ + host=None, port=None, *, ssl=None, \ family=0, proto=0, flags=0, sock=None, \ local_addr=None, server_hostname=None, \ - ssl_handshake_timeout=None) + ssl_handshake_timeout=None, \ + happy_eyeballs_delay=None, interleave=None) Open a streaming transport connection to a given address specified by *host* and *port*. @@ -430,7 +437,18 @@ Opening network connections .. versionadded:: 3.8 - The *happy_eyeballs_delay* and *interleave* parameters. + Added the *happy_eyeballs_delay* and *interleave* parameters. + + Happy Eyeballs Algorithm: Success with Dual-Stack Hosts. + When a server's IPv4 path and protocol are working, but the server's + IPv6 path and protocol are not working, a dual-stack client + application experiences significant connection delay compared to an + IPv4-only client. This is undesirable because it causes the dual- + stack client to have a worse user experience. This document + specifies requirements for algorithms that reduce this user-visible + delay and provides an algorithm. + + For more information: https://tools.ietf.org/html/rfc6555 .. versionadded:: 3.7 @@ -452,11 +470,26 @@ Opening network connections that can be used directly in async/await code. .. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ - local_addr=None, remote_addr=None, \*, \ + local_addr=None, remote_addr=None, *, \ family=0, proto=0, flags=0, \ reuse_address=None, reuse_port=None, \ allow_broadcast=None, sock=None) + .. note:: + The parameter *reuse_address* is no longer supported, as using + :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for + UDP. Explicitly passing ``reuse_address=True`` will raise an exception. + + When multiple processes with differing UIDs assign sockets to an + identical UDP socket address with ``SO_REUSEADDR``, incoming packets can + become randomly distributed among the sockets. + + For supported platforms, *reuse_port* can be used as a replacement for + similar functionality. With *reuse_port*, + :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically + prevents processes with differing UIDs from assigning sockets to the same + socket address. + Create a datagram connection. The socket family can be either :py:data:`~socket.AF_INET`, @@ -485,11 +518,6 @@ Opening network connections resolution. If given, these should all be integers from the corresponding :mod:`socket` module constants. - * *reuse_address* tells the kernel to reuse a local socket in - ``TIME_WAIT`` state, without waiting for its natural timeout to - expire. If not specified will automatically be set to ``True`` on - Unix. - * *reuse_port* tells the kernel to allow this endpoint to be bound to the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows @@ -511,11 +539,15 @@ Opening network connections The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, *allow_broadcast*, and *sock* parameters were added. + .. versionchanged:: 3.8.1 + The *reuse_address* parameter is no longer supported due to security + concerns. + .. versionchanged:: 3.8 Added support for Windows. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ - path=None, \*, ssl=None, sock=None, \ + path=None, *, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) Create a Unix connection. @@ -548,7 +580,7 @@ Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_server(protocol_factory, \ - host=None, port=None, \*, \ + host=None, port=None, *, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ @@ -639,7 +671,7 @@ Creating network servers .. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ - \*, sock=None, backlog=100, ssl=None, \ + *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, start_serving=True) Similar to :meth:`loop.create_server` but works with the @@ -664,7 +696,7 @@ Creating network servers The *path* parameter can now be a :class:`~pathlib.Path` object. .. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ - sock, \*, ssl=None, ssl_handshake_timeout=None) + sock, *, ssl=None, ssl_handshake_timeout=None) Wrap an already accepted connection into a transport/protocol pair. @@ -729,7 +761,7 @@ TLS Upgrade ^^^^^^^^^^^ .. coroutinemethod:: loop.start_tls(transport, protocol, \ - sslcontext, \*, server_side=False, \ + sslcontext, *, server_side=False, \ server_hostname=None, ssl_handshake_timeout=None) Upgrade an existing transport-based connection to TLS. @@ -762,7 +794,7 @@ TLS Upgrade Watching file descriptors ^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: loop.add_reader(fd, callback, \*args) +.. method:: loop.add_reader(fd, callback, *args) Start monitoring the *fd* file descriptor for read availability and invoke *callback* with the specified arguments once *fd* is available for @@ -772,7 +804,7 @@ Watching file descriptors Stop monitoring the *fd* file descriptor for read availability. -.. method:: loop.add_writer(fd, callback, \*args) +.. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and invoke *callback* with the specified arguments once *fd* is available for @@ -886,7 +918,7 @@ convenient. :meth:`loop.create_server` and :func:`start_server`. .. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ - \*, fallback=True) + *, fallback=True) Send a file using high-performance :mod:`os.sendfile` if possible. Return the total number of bytes sent. @@ -920,7 +952,7 @@ convenient. DNS ^^^ -.. coroutinemethod:: loop.getaddrinfo(host, port, \*, family=0, \ +.. coroutinemethod:: loop.getaddrinfo(host, port, *, family=0, \ type=0, proto=0, flags=0) Asynchronous version of :meth:`socket.getaddrinfo`. @@ -985,7 +1017,7 @@ Working with pipes Unix signals ^^^^^^^^^^^^ -.. method:: loop.add_signal_handler(signum, callback, \*args) +.. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1020,7 +1052,7 @@ Unix signals Executing code in thread or process pools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. awaitablemethod:: loop.run_in_executor(executor, func, \*args) +.. awaitablemethod:: loop.run_in_executor(executor, func, *args) Arrange for *func* to be called in the specified executor. @@ -1190,9 +1222,9 @@ async/await code consider using the high-level subprocesses. See :ref:`Subprocess Support on Windows ` for details. -.. coroutinemethod:: loop.subprocess_exec(protocol_factory, \*args, \ +.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from one or more string arguments specified by *args*. @@ -1272,9 +1304,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. -.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, \*, \ +.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from *cmd*, which can be a :class:`str` or a :class:`bytes` string encoded to the @@ -1454,7 +1486,7 @@ asyncio ships with two different event loop implementations: :class:`SelectorEventLoop` and :class:`ProactorEventLoop`. By default asyncio is configured to use :class:`SelectorEventLoop` -on all platforms. +on Unix and :class:`ProactorEventLoop` on Windows. .. class:: SelectorEventLoop diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst index e49577a203e8fb..7166d5c4bd88f9 100644 --- a/Doc/library/asyncio-exceptions.rst +++ b/Doc/library/asyncio-exceptions.rst @@ -7,6 +7,9 @@ Exceptions ========== +**Source code:** :source:`Lib/asyncio/exceptions.py` + +---------------------------------------------------- .. exception:: TimeoutError @@ -25,26 +28,9 @@ Exceptions when asyncio Tasks are cancelled. In almost all situations the exception must be re-raised. - .. important:: - - This exception is a subclass of :exc:`Exception`, so it can be - accidentally suppressed by an overly broad ``try..except`` block:: - - try: - await operation - except Exception: - # The cancellation is broken because the *except* block - # suppresses the CancelledError exception. - log.log('an error has occurred') - - Instead, the following pattern should be used:: + .. versionchanged:: 3.8 - try: - await operation - except asyncio.CancelledError: - raise - except Exception: - log.log('an error has occurred') + :exc:`CancelledError` is now a subclass of :class:`BaseException`. .. exception:: InvalidStateError diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 6e6e0137c1bdda..45a2754e9658d9 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -7,6 +7,11 @@ Futures ======= +**Source code:** :source:`Lib/asyncio/futures.py`, +:source:`Lib/asyncio/base_futures.py` + +------------------------------------- + *Future* objects are used to bridge **low-level callback-based code** with high-level async/await code. @@ -26,7 +31,7 @@ Future Functions .. versionadded:: 3.5 -.. function:: ensure_future(obj, \*, loop=None) +.. function:: ensure_future(obj, *, loop=None) Return: @@ -35,7 +40,9 @@ Future Functions is used for the test.) * a :class:`Task` object wrapping *obj*, if *obj* is a - coroutine (:func:`iscoroutine` is used for the test.) + coroutine (:func:`iscoroutine` is used for the test); + in this case the coroutine will be scheduled by + ``ensure_future()``. * a :class:`Task` object that would await on *obj*, if *obj* is an awaitable (:func:`inspect.isawaitable` is used for the test.) @@ -51,7 +58,7 @@ Future Functions The function accepts any :term:`awaitable` object. -.. function:: wrap_future(future, \*, loop=None) +.. function:: wrap_future(future, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. @@ -60,7 +67,7 @@ Future Functions Future Object ============= -.. class:: Future(\*, loop=None) +.. class:: Future(*, loop=None) A Future represents an eventual result of an asynchronous operation. Not thread-safe. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 7e4a70f91c6ed5..390ee1969d096f 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -23,6 +23,12 @@ All Platforms Windows ======= +**Source code:** :source:`Lib/asyncio/proactor_events.py`, +:source:`Lib/asyncio/windows_events.py`, +:source:`Lib/asyncio/windows_utils.py` + +-------------------------------------- + .. versionchanged:: 3.8 On Windows, :class:`ProactorEventLoop` is now the default event loop. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 6212df85dbc10a..ceb2323c54586f 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -117,6 +117,7 @@ asyncio ships with the following built-in policies: .. availability:: Windows. +.. _asyncio-watchers: Process Watchers ================ @@ -129,10 +130,11 @@ In asyncio, child processes are created with :func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` functions. -asyncio defines the :class:`AbstractChildWatcher` abstract base class, -which child watchers should implement, and has two different -implementations: :class:`SafeChildWatcher` (configured to be used -by default) and :class:`FastChildWatcher`. +asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child +watchers should implement, and has four different implementations: +:class:`ThreadedChildWatcher` (configured to be used by default), +:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and +:class:`FastChildWatcher`. See also the :ref:`Subprocess and Threads ` section. @@ -157,7 +159,7 @@ implementation used by the asyncio event loop: .. class:: AbstractChildWatcher - .. method:: add_child_handler(pid, callback, \*args) + .. method:: add_child_handler(pid, callback, *args) Register a new child handler. @@ -184,6 +186,15 @@ implementation used by the asyncio event loop: Note: loop may be ``None``. + .. method:: is_active() + + Return ``True`` if the watcher is ready to use. + + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. + + .. versionadded:: 3.8 + .. method:: close() Close the watcher. @@ -191,16 +202,48 @@ implementation used by the asyncio event loop: This method has to be called to ensure that underlying resources are cleaned-up. -.. class:: SafeChildWatcher +.. class:: ThreadedChildWatcher + + This implementation starts a new waiting thread for every subprocess spawn. + + It works reliably even when the asyncio event loop is run in a non-main OS thread. + + There is no noticeable overhead when handling a big number of children (*O(1)* each + time a child terminates), but starting a thread per process requires extra memory. + + This watcher is used by default. + + .. versionadded:: 3.8 - This implementation avoids disrupting other code spawning processes +.. class:: MultiLoopChildWatcher + + This implementation registers a :py:data:`SIGCHLD` signal handler on + instantiation. That can break third-party code that installs a custom handler for + :py:data:`SIGCHLD` signal. + + The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. - This is a safe solution but it has a significant overhead when + There is no limitation for running subprocesses from different threads once the + watcher is installed. + + The solution is safe but it has a significant overhead when handling a big number of processes (*O(n)* each time a :py:data:`SIGCHLD` is received). - asyncio uses this safe implementation by default. + .. versionadded:: 3.8 + +.. class:: SafeChildWatcher + + This implementation uses active event loop from the main thread to handle + :py:data:`SIGCHLD` signal. If the main thread has no running event loop another + thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). + + The watcher avoids disrupting other code spawning processes + by polling every process explicitly on a :py:data:`SIGCHLD` signal. + + This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* + complexity but requires a running event loop in the main thread to work. .. class:: FastChildWatcher @@ -211,6 +254,9 @@ implementation used by the asyncio event loop: There is no noticeable overhead when handling a big number of children (*O(1)* each time a child terminates). + This solution requires a running event loop in the main thread to work, as + :class:`SafeChildWatcher`. + Custom Policies =============== diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index f08738dd62bb5c..816ddcd03b0086 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -69,6 +69,10 @@ This documentation page contains the following sections: Transports ========== +**Source code:** :source:`Lib/asyncio/transports.py` + +---------------------------------------------------- + Transports are classes provided by :mod:`asyncio` in order to abstract various kinds of communication channels. @@ -431,6 +435,10 @@ Subprocess Transports Protocols ========= +**Source code:** :source:`Lib/asyncio/protocols.py` + +--------------------------------------------------- + asyncio provides a set of abstract base classes that should be used to implement network protocols. Those classes are meant to be used together with :ref:`transports `. @@ -767,9 +775,8 @@ data, and waits until the connection is closed:: class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, on_con_lost, loop): + def __init__(self, message, on_con_lost): self.message = message - self.loop = loop self.on_con_lost = on_con_lost def connection_made(self, transport): @@ -793,7 +800,7 @@ data, and waits until the connection is closed:: message = 'Hello World!' transport, protocol = await loop.create_connection( - lambda: EchoClientProtocol(message, on_con_lost, loop), + lambda: EchoClientProtocol(message, on_con_lost), '127.0.0.1', 8888) # Wait until the protocol signals that the connection @@ -869,11 +876,10 @@ method, sends data and closes the transport when it receives the answer:: class EchoClientProtocol: - def __init__(self, message, loop): + def __init__(self, message, on_con_lost): self.message = message - self.loop = loop + self.on_con_lost = on_con_lost self.transport = None - self.on_con_lost = loop.create_future() def connection_made(self, transport): self.transport = transport @@ -899,13 +905,15 @@ method, sends data and closes the transport when it receives the answer:: # low-level APIs. loop = asyncio.get_running_loop() + on_con_lost = loop.create_future() message = "Hello World!" + transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, loop), + lambda: EchoClientProtocol(message, on_con_lost), remote_addr=('127.0.0.1', 9999)) try: - await protocol.on_con_lost + await on_con_lost finally: transport.close() @@ -927,9 +935,9 @@ Wait until a socket receives data using the class MyProtocol(asyncio.Protocol): - def __init__(self, loop): + def __init__(self, on_con_lost): self.transport = None - self.on_con_lost = loop.create_future() + self.on_con_lost = on_con_lost def connection_made(self, transport): self.transport = transport @@ -950,13 +958,14 @@ Wait until a socket receives data using the # Get a reference to the event loop as we plan to use # low-level APIs. loop = asyncio.get_running_loop() + on_con_lost = loop.create_future() # Create a pair of connected sockets rsock, wsock = socket.socketpair() # Register the socket to wait for data. transport, protocol = await loop.create_connection( - lambda: MyProtocol(loop), sock=rsock) + lambda: MyProtocol(on_con_lost), sock=rsock) # Simulate the reception of data from the network. loop.call_soon(wsock.send, 'abc'.encode()) @@ -987,7 +996,7 @@ loop.subprocess_exec() and SubprocessProtocol An example of a subprocess protocol used to get the output of a subprocess and to wait for the subprocess exit. -The subprocess is created by th :meth:`loop.subprocess_exec` method:: +The subprocess is created by the :meth:`loop.subprocess_exec` method:: import asyncio import sys diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 7be1023c80cc66..524560b691d720 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -6,6 +6,10 @@ Queues ====== +**Source code:** :source:`Lib/asyncio/queues.py` + +------------------------------------------------ + asyncio queues are designed to be similar to classes of the :mod:`queue` module. Although asyncio queues are not thread-safe, they are designed to be used specifically in async/await code. @@ -32,6 +36,10 @@ Queue the queue is always known and can be returned by calling the :meth:`qsize` method. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + + This class is :ref:`not thread safe `. .. attribute:: maxsize diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 28ca5d5f339692..714de8d41a3503 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -6,6 +6,10 @@ Streams ======= +**Source code:** :source:`Lib/asyncio/streams.py` + +------------------------------------------------- + Streams are high-level async/await-ready primitives to work with network connections. Streams allow sending and receiving data without using callbacks or low-level protocols and transports. @@ -22,13 +26,15 @@ streams:: '127.0.0.1', 8888) print(f'Send: {message!r}') - await writer.write(message.encode()) + writer.write(message.encode()) + await writer.drain() data = await reader.read(100) print(f'Received: {data.decode()!r}') print('Close the connection') - await writer.close() + writer.close() + await writer.wait_closed() asyncio.run(tcp_echo_client('Hello World!')) @@ -42,7 +48,7 @@ The following top-level asyncio functions can be used to create and work with streams: -.. coroutinefunction:: open_connection(host=None, port=None, \*, \ +.. coroutinefunction:: open_connection(host=None, port=None, *, \ loop=None, limit=None, ssl=None, family=0, \ proto=0, flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -67,12 +73,8 @@ and work with streams: The *ssl_handshake_timeout* parameter. - .. deprecated-removed:: 3.8 3.10 - - `open_connection()` is deprecated in favor of `connect()`. - .. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, \*, loop=None, limit=None, \ + port=None, *, loop=None, limit=None, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -104,14 +106,10 @@ and work with streams: The *ssl_handshake_timeout* and *start_serving* parameters. - .. deprecated-removed:: 3.8 3.10 - - `start_server()` is deprecated if favor of `StreamServer()` - .. rubric:: Unix Sockets -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ +.. coroutinefunction:: open_unix_connection(path=None, *, loop=None, \ limit=None, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -132,13 +130,9 @@ and work with streams: The *path* parameter can now be a :term:`path-like object` - .. deprecated-removed:: 3.8 3.10 - - `open_unix_connection()` is deprecated if favor of `connect_unix()`. - .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ - \*, loop=None, limit=None, sock=None, \ + *, loop=None, limit=None, sock=None, \ backlog=100, ssl=None, ssl_handshake_timeout=None, \ start_serving=True) @@ -158,13 +152,6 @@ and work with streams: The *path* parameter can now be a :term:`path-like object`. - .. deprecated-removed:: 3.8 3.10 - - `start_unix_server()` is deprecated in favor of `UnixStreamServer()`. - - ---------- - StreamReader ============ @@ -248,22 +235,11 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. - Starting with Python 3.8, it is possible to directly await on the `write()` - method:: - - await stream.write(data) - - The ``await`` pauses the current coroutine until the data is written to the - socket. - - Below is an equivalent code that works with Python <= 3.7:: + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() - .. versionchanged:: 3.8 - Support ``await stream.write(...)`` syntax. - .. method:: writelines(data) The method writes a list (or any iterable) of bytes to the underlying socket @@ -271,46 +247,24 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. - Starting with Python 3.8, it is possible to directly await on the `write()` - method:: - - await stream.writelines(lines) - - The ``await`` pauses the current coroutine until the data is written to the - socket. - - Below is an equivalent code that works with Python <= 3.7:: + The method should be used along with the ``drain()`` method:: stream.writelines(lines) await stream.drain() - .. versionchanged:: 3.8 - Support ``await stream.writelines()`` syntax. - .. method:: close() The method closes the stream and the underlying socket. - Starting with Python 3.8, it is possible to directly await on the `close()` - method:: - - await stream.close() - - The ``await`` pauses the current coroutine until the stream and the underlying - socket are closed (and SSL shutdown is performed for a secure connection). - - Below is an equivalent code that works with Python <= 3.7:: + The method should be used along with the ``wait_closed()`` method:: stream.close() await stream.wait_closed() - .. versionchanged:: 3.8 - Support ``await stream.close()`` syntax. - .. method:: can_write_eof() - Return *True* if the underlying transport supports - the :meth:`write_eof` method, *False* otherwise. + Return ``True`` if the underlying transport supports + the :meth:`write_eof` method, ``False`` otherwise. .. method:: write_eof() diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 00dc66c48b2189..6ba24249f28d84 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -6,6 +6,11 @@ Subprocesses ============ +**Source code:** :source:`Lib/asyncio/subprocess.py`, +:source:`Lib/asyncio/base_subprocess.py` + +---------------------------------------- + This section describes high-level async/await asyncio APIs to create and manage subprocesses. @@ -56,9 +61,9 @@ See also the `Examples`_ subsection. Creating Subprocesses ===================== -.. coroutinefunction:: create_subprocess_exec(program, \*args, stdin=None, \ +.. coroutinefunction:: create_subprocess_exec(program, *args, stdin=None, \ stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) + limit=None, **kwds) Create a subprocess. @@ -71,9 +76,13 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. + .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) + limit=None, **kwds) Run the *cmd* shell command. @@ -86,21 +95,23 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. -.. important:: + .. important:: + + It is the application's responsibility to ensure that all whitespace and + special characters are quoted appropriately to avoid `shell injection + `_ + vulnerabilities. The :func:`shlex.quote` function can be used to properly + escape whitespace and special shell characters in strings that are going + to be used to construct shell commands. - It is the application's responsibility to ensure that all whitespace and - special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and special shell characters in strings that are going - to be used to construct shell commands. + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. .. note:: - The default asyncio event loop implementation on **Windows** does not - support subprocesses. Subprocesses are available for Windows if a - :class:`ProactorEventLoop` is used. - See :ref:`Subprocess Support on Windows ` + Subprocesses are available for Windows if a :class:`ProactorEventLoop` is + used. See :ref:`Subprocess Support on Windows ` for details. .. seealso:: @@ -240,7 +251,7 @@ their completion. .. method:: kill() - Kill the child. + Kill the child process. On POSIX systems this method sends :py:data:`SIGKILL` to the child process. @@ -293,18 +304,26 @@ their completion. Subprocess and Threads ---------------------- -Standard asyncio event loop supports running subprocesses from -different threads, but there are limitations: +Standard asyncio event loop supports running subprocesses from different threads by +default. + +On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), +:class:`SelectorEventLoop` has no subprocess support. + +On UNIX *child watchers* are used for subprocess finish waiting, see +:ref:`asyncio-watchers` for more info. + + +.. versionchanged:: 3.8 -* An event loop must run in the main thread. + UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from + different threads without any limitation. -* The child watcher must be instantiated in the main thread - before executing subprocesses from other threads. Call the - :func:`get_child_watcher` function in the main thread to instantiate - the child watcher. + Spawning a subprocess with *inactive* current child watcher raises + :exc:`RuntimeError`. -Note that alternative event loop implementations might not share -the above limitations; please refer to their documentation. +Note that alternative event loop implementations might have own limitations; +please refer to their documentation. .. seealso:: diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 79f6b02d85e2e5..c401fb50a50751 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -6,6 +6,10 @@ Synchronization Primitives ========================== +**Source code:** :source:`Lib/asyncio/locks.py` + +----------------------------------------------- + asyncio synchronization primitives are designed to be similar to those of the :mod:`threading` module with two important caveats: @@ -59,6 +63,9 @@ Lock finally: lock.release() + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. coroutinemethod:: acquire() Acquire the lock. @@ -97,10 +104,14 @@ Event that some event has happened. An Event object manages an internal flag that can be set to *true* - with the :meth:`set` method and reset to *false* with the - :meth:`clear` method. The :meth:`wait` method blocks until the + with the :meth:`~Event.set` method and reset to *false* with the + :meth:`clear` method. The :meth:`~Event.wait` method blocks until the flag is set to *true*. The flag is set to *false* initially. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. _asyncio_example_sync_event: Example:: @@ -131,7 +142,7 @@ Event Wait until the event is set. If the event is set, return ``True`` immediately. - Otherwise block until another task calls :meth:`set`. + Otherwise block until another task calls :meth:`~Event.set`. .. method:: set() @@ -144,8 +155,8 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`wait` will now block until the - :meth:`set` method is called again. + Tasks awaiting on :meth:`~Event.wait` will now block until the + :meth:`~Event.set` method is called again. .. method:: is_set() @@ -173,6 +184,10 @@ Condition ``None``. In the latter case a new Lock object is created automatically. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + The preferred way to use a Condition is an :keyword:`async with` statement:: @@ -269,6 +284,10 @@ Semaphore internal counter (``1`` by default). If the given value is less than ``0`` a :exc:`ValueError` is raised. + + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + The preferred way to use a Semaphore is an :keyword:`async with` statement:: @@ -322,6 +341,9 @@ BoundedSemaphore increases the internal counter above the initial *value*. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + --------- diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 1fcdcb985d8842..246d5b9a73c548 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -18,9 +18,9 @@ and Tasks. Coroutines ========== -Coroutines declared with async/await syntax is the preferred way of -writing asyncio applications. For example, the following snippet -of code (requires Python 3.7+) prints "hello", waits 1 second, +:term:`Coroutines ` declared with the async/await syntax is the +preferred way of writing asyncio applications. For example, the following +snippet of code (requires Python 3.7+) prints "hello", waits 1 second, and then prints "world":: >>> import asyncio @@ -210,7 +210,9 @@ is :meth:`loop.run_in_executor`. Running an asyncio Program ========================== -.. function:: run(coro, \*, debug=False) +.. function:: run(coro, *, debug=False) + + Execute the :term:`coroutine` *coro* and return the result. This function runs the passed coroutine, taking care of managing the asyncio event loop and *finalizing asynchronous @@ -225,15 +227,25 @@ Running an asyncio Program the end. It should be used as a main entry point for asyncio programs, and should ideally only be called once. + Example:: + + async def main(): + await asyncio.sleep(1) + print('hello') + + asyncio.run(main()) + .. versionadded:: 3.7 - **Important:** this function has been added to asyncio in - Python 3.7 on a :term:`provisional basis `. + .. note:: + The source code for ``asyncio.run()`` can be found in + :source:`Lib/asyncio/runners.py`. + Creating Tasks ============== -.. function:: create_task(coro, \*, name=None) +.. function:: create_task(coro, *, name=None) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. @@ -305,7 +317,7 @@ Sleeping Running Tasks Concurrently ========================== -.. awaitablefunction:: gather(\*aws, loop=None, return_exceptions=False) +.. awaitablefunction:: gather(*aws, loop=None, return_exceptions=False) Run :ref:`awaitable objects ` in the *aws* sequence *concurrently*. @@ -334,6 +346,9 @@ Running Tasks Concurrently cancellation of one submitted Task/Future to cause other Tasks/Futures to be cancelled. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. _asyncio_example_gather: Example:: @@ -370,6 +385,14 @@ Running Tasks Concurrently # Task C: Compute factorial(4)... # Task C: factorial(4) = 24 + .. note:: + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. + .. versionchanged:: 3.7 If the *gather* itself is cancelled, the cancellation is propagated regardless of *return_exceptions*. @@ -411,6 +434,9 @@ Shielding From Cancellation except CancelledError: res = None + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + Timeouts ======== @@ -471,29 +497,19 @@ Timeouts Waiting Primitives ================== -.. coroutinefunction:: wait(aws, \*, loop=None, timeout=None,\ +.. coroutinefunction:: wait(aws, *, loop=None, timeout=None,\ return_when=ALL_COMPLETED) Run :ref:`awaitable objects ` in the *aws* - set concurrently and block until the condition specified + iterable concurrently and block until the condition specified by *return_when*. - .. deprecated:: 3.8 - - If any awaitable in *aws* is a coroutine, it is automatically - scheduled as a Task. Passing coroutines objects to - ``wait()`` directly is deprecated as it leads to - :ref:`confusing behavior `. - Returns two sets of Tasks/Futures: ``(done, pending)``. Usage:: done, pending = await asyncio.wait(aws) - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - *timeout* (a float or int), if specified, can be used to control the maximum number of seconds to wait before returning. @@ -525,6 +541,17 @@ Waiting Primitives Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. + .. deprecated:: 3.8 + + If any awaitable in *aws* is a coroutine, it is automatically + scheduled as a Task. Passing coroutines objects to + ``wait()`` directly is deprecated as it leads to + :ref:`confusing behavior `. + + .. deprecated-removed:: 3.8 3.10 + + The *loop* parameter. + .. _asyncio_example_wait_coroutine: .. note:: @@ -558,20 +585,23 @@ Waiting Primitives deprecated. -.. function:: as_completed(aws, \*, loop=None, timeout=None) +.. function:: as_completed(aws, *, loop=None, timeout=None) Run :ref:`awaitable objects ` in the *aws* - set concurrently. Return an iterator of :class:`Future` objects. - Each Future object returned represents the earliest result - from the set of the remaining awaitables. + iterable concurrently. Return an iterator of coroutines. + Each coroutine returned can be awaited to get the earliest next + result from the iterable of the remaining awaitables. Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures are done. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + Example:: - for f in as_completed(aws): - earliest_result = await f + for coro in as_completed(aws): + earliest_result = await coro # ... @@ -649,7 +679,7 @@ Introspection Task Object =========== -.. class:: Task(coro, \*, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -694,6 +724,9 @@ Task Object .. versionchanged:: 3.8 Added the ``name`` parameter. + .. deprecated-removed:: 3.8 3.10 + The *loop* parameter. + .. method:: cancel() Request the Task to be cancelled. @@ -809,7 +842,7 @@ Task Object See the documentation of :meth:`Future.remove_done_callback` for more details. - .. method:: get_stack(\*, limit=None) + .. method:: get_stack(*, limit=None) Return the list of stack frames for this Task. @@ -830,7 +863,7 @@ Task Object stack are returned, but the oldest frames of a traceback are returned. (This matches the behavior of the traceback module.) - .. method:: print_stack(\*, limit=None, file=None) + .. method:: print_stack(*, limit=None, file=None) Print the stack or traceback for this Task. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 6990adb21e3603..94a853259d3483 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -4,7 +4,7 @@ .. module:: asyncio :synopsis: Asynchronous I/O. --------------- +------------------------------- .. sidebar:: Hello World! @@ -91,3 +91,6 @@ Additionally, there are **low-level** APIs for asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + +.. note:: + The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst new file mode 100644 index 00000000000000..8227a7955bef81 --- /dev/null +++ b/Doc/library/audit_events.rst @@ -0,0 +1,47 @@ +.. _audit-events: + +.. index:: single: audit events + +Audit events table +================== + +This table contains all events raised by :func:`sys.audit` or +:c:func:`PySys_Audit` calls throughout the CPython runtime and the +standard library. These calls were added in 3.8.0 or later (see :pep:`578`). + +See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for +information on handling these events. + +.. impl-detail:: + + This table is generated from the CPython documentation, and may not + represent events raised by other implementations. See your runtime + specific documentation for actual events raised. + +.. audit-event-table:: + +The following events are raised internally and do not correspond to any +public API of CPython: + ++--------------------------+-------------------------------------------+ +| Audit event | Arguments | ++==========================+===========================================+ +| _winapi.CreateFile | ``file_name``, ``desired_access``, | +| | ``share_mode``, ``creation_disposition``, | +| | ``flags_and_attributes`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreateJunction | ``src_path``, ``dst_path`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreateNamedPipe | ``name``, ``open_mode``, ``pipe_mode`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreatePipe | | ++--------------------------+-------------------------------------------+ +| _winapi.CreateProcess | ``application_name``, ``command_line``, | +| | ``current_directory`` | ++--------------------------+-------------------------------------------+ +| _winapi.OpenProcess | ``process_id``, ``desired_access`` | ++--------------------------+-------------------------------------------+ +| _winapi.TerminateProcess | ``handle``, ``exit_code`` | ++--------------------------+-------------------------------------------+ +| ctypes.PyObj_FromPtr | ``obj`` | ++--------------------------+-------------------------------------------+ diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 277de601cb7b6f..cb9af5839fd471 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -264,7 +264,6 @@ Below are some examples of typical usage of the :mod:`bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -273,11 +272,9 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> c = bz2.compress(data) >>> len(data) / len(c) # Data compression ratio 1.513595166163142 - >>> d = bz2.decompress(c) >>> data == d # Check equality to original object after round-trip True @@ -285,7 +282,6 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress Using :class:`BZ2Compressor` for incremental compression: >>> import bz2 - >>> def gen_data(chunks=10, chunksize=1000): ... """Yield incremental blocks of chunksize bytes.""" ... for _ in range(chunks): @@ -308,7 +304,6 @@ while ordered, repetitive data usually yields a high compression ratio. Writing and reading a bzip2-compressed file in binary mode: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -317,14 +312,11 @@ Writing and reading a bzip2-compressed file in binary mode: ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> with bz2.open("myfile.bz2", "wb") as f: ... # Write compressed data to file ... unused = f.write(data) - >>> with bz2.open("myfile.bz2", "rb") as f: ... # Decompress data from file ... content = f.read() - >>> content == data # Check equality to original object after round-trip True diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 56b75ef0f850a6..c3c04db853ed2d 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -349,7 +349,7 @@ For simple text calendars this module provides the following functions. .. function:: monthcalendar(year, month) Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month a represented by zeros. Each week begins with Monday + days outside of the month are represented by zeros. Each week begins with Monday unless set by :func:`setfirstweekday`. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index 4048592e7361f7..880074bed60264 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -277,14 +277,16 @@ These are useful if you want more control, or if you want to employ some of the algorithms implemented in this module in other circumstances. -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False) +.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are + ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are passed to :func:`urllib.parse.parse_qs` unchanged. + .. versionchanged:: 3.8.8 + Added the *separator* parameter. -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace") +.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") Parse input of type :mimetype:`multipart/form-data` (for file uploads). Arguments are *fp* for the input file, *pdict* for a dictionary containing @@ -303,6 +305,9 @@ algorithms implemented in this module in other circumstances. Added the *encoding* and *errors* parameters. For non-file fields, the value is now a list of strings, not bytes. + .. versionchanged:: 3.8.8 + Added the *separator* parameter. + .. function:: parse_header(string) diff --git a/Doc/library/code.rst b/Doc/library/code.rst index e2c47bab5a0b91..538e5afc7822aa 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -56,8 +56,8 @@ build applications which provide an interactive interpreter prompt. *source* is the source string; *filename* is the optional filename from which source was read, defaulting to ``''``; and *symbol* is the optional - grammar start symbol, which should be either ``'single'`` (the default) or - ``'eval'``. + grammar start symbol, which should be ``'single'`` (the default), ``'eval'`` + or ``'exec'``. Returns a code object (the same as ``compile(source, filename, symbol)``) if the command is complete and valid; ``None`` if the command is incomplete; raises @@ -76,7 +76,7 @@ Interactive Interpreter Objects Compile and run some source in the interpreter. Arguments are the same as for :func:`compile_command`; the default for *filename* is ``''``, and for - *symbol* is ``'single'``. One several things can happen: + *symbol* is ``'single'``. One of several things can happen: * The input is incorrect; :func:`compile_command` raised an exception (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2e9314e0fab793..f09b0e1abdc423 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -292,7 +292,7 @@ Error Handlers To simplify and standardize error handling, codecs may implement different error handling schemes by -accepting the *errors* string argument. The following string values are +accepting the *errors* string argument. The following string values are defined and implemented by all standard Python codecs: .. tabularcolumns:: |l|L| @@ -301,11 +301,11 @@ defined and implemented by all standard Python codecs: | Value | Meaning | +=========================+===============================================+ | ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass); | -| | this is the default. Implemented in | +| | this is the default. Implemented in | | | :func:`strict_errors`. | +-------------------------+-----------------------------------------------+ | ``'ignore'`` | Ignore the malformed data and continue | -| | without further notice. Implemented in | +| | without further notice. Implemented in | | | :func:`ignore_errors`. | +-------------------------+-----------------------------------------------+ @@ -327,11 +327,11 @@ The following error handlers are only applicable to | | marker; Python will use the official | | | ``U+FFFD`` REPLACEMENT CHARACTER for the | | | built-in codecs on decoding, and '?' on | -| | encoding. Implemented in | +| | encoding. Implemented in | | | :func:`replace_errors`. | +-------------------------+-----------------------------------------------+ | ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | -| | reference (only for encoding). Implemented | +| | reference (only for encoding). Implemented | | | in :func:`xmlcharrefreplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'backslashreplace'`` | Replace with backslashed escape sequences. | @@ -339,15 +339,15 @@ The following error handlers are only applicable to | | :func:`backslashreplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'namereplace'`` | Replace with ``\N{...}`` escape sequences | -| | (only for encoding). Implemented in | +| | (only for encoding). Implemented in | | | :func:`namereplace_errors`. | +-------------------------+-----------------------------------------------+ | ``'surrogateescape'`` | On decoding, replace byte with individual | | | surrogate code ranging from ``U+DC80`` to | -| | ``U+DCFF``. This code will then be turned | +| | ``U+DCFF``. This code will then be turned | | | back into the same byte when the | | | ``'surrogateescape'`` error handler is used | -| | when encoding the data. (See :pep:`383` for | +| | when encoding the data. (See :pep:`383` for | | | more.) | +-------------------------+-----------------------------------------------+ @@ -357,7 +357,7 @@ In addition, the following error handler is specific to the given codecs: | Value | Codecs | Meaning | +===================+========================+===========================================+ |``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | -| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | +| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | | | utf-32-be, utf-32-le | presence of surrogates as an error. | +-------------------+------------------------+-------------------------------------------+ @@ -388,9 +388,9 @@ handler: error handler must either raise this or a different exception, or return a tuple with a replacement for the unencodable part of the input and a position where encoding should continue. The replacement may be either :class:`str` or - :class:`bytes`. If the replacement is bytes, the encoder will simply copy + :class:`bytes`. If the replacement is bytes, the encoder will simply copy them into the output buffer. If the replacement is a string, the encoder will - encode the replacement. Encoding continues on original input at the + encode the replacement. Encoding continues on original input at the specified position. Negative position values will be treated as being relative to the end of the input string. If the resulting position is out of bound an :exc:`IndexError` will be raised. @@ -484,7 +484,7 @@ function interfaces of the stateless encoder and decoder: .. method:: Codec.decode(input[, errors]) Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts + consumed). For instance, for a :term:`text encoding`, decoding converts a bytes object encoded using a particular character set encoding to a string object. @@ -568,7 +568,7 @@ define in order to be compatible with the Python codec registry. implementation should make sure that ``0`` is the most common state. (States that are more complicated than integers can be converted into an integer by marshaling/pickling the state and encoding the bytes - of the resulting string into an integer). + of the resulting string into an integer.) .. method:: setstate(state) @@ -751,7 +751,7 @@ compatible with the Python codec registry. number of encoded bytes or code points to read for decoding. The decoder can modify this setting as appropriate. The default value -1 indicates to read and decode as much as - possible. This parameter is intended to + possible. This parameter is intended to prevent having to decode huge files in one step. The *firstline* flag indicates that @@ -780,8 +780,8 @@ compatible with the Python codec registry. Read all lines available on the input stream and return them as a list of lines. - Line-endings are implemented using the codec's decoder method and are - included in the list entries if *keepends* is true. + Line-endings are implemented using the codec's :meth:`decode` method and + are included in the list entries if *keepends* is true. *sizehint*, if given, is passed as the *size* argument to the stream's :meth:`read` method. @@ -791,7 +791,7 @@ compatible with the Python codec registry. Resets the codec buffers used for keeping state. - Note that no stream repositioning should take place. This method is + Note that no stream repositioning should take place. This method is primarily intended to be able to recover from decoding errors. @@ -841,7 +841,7 @@ The design is such that one can use the factory functions returned by the code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* work on the backend — the data in *stream*. - You can use these objects to do transparent transcodings from e.g. Latin-1 + You can use these objects to do transparent transcodings, e.g., from Latin-1 to UTF-8 and back. The *stream* argument must be a file-like object. @@ -866,10 +866,10 @@ Encodings and Unicode --------------------- Strings are stored internally as sequences of code points in -range ``0x0``--``0x10FFFF``. (See :pep:`393` for +range ``0x0``--``0x10FFFF``. (See :pep:`393` for more details about the implementation.) Once a string object is used outside of CPU and memory, endianness -and how these arrays are stored as bytes become an issue. As with other +and how these arrays are stored as bytes become an issue. As with other codecs, serialising a string into a sequence of bytes is known as *encoding*, and recreating the string from the sequence of bytes is known as *decoding*. There are a variety of different text serialisation codecs, which are @@ -964,7 +964,7 @@ to determine the byte order used for generating the byte sequence, but as a signature that helps in guessing the encoding. On encoding the utf-8-sig codec will write ``0xef``, ``0xbb``, ``0xbf`` as the first three bytes to the file. On decoding ``utf-8-sig`` will skip those three bytes if they appear as the first -three bytes in the file. In UTF-8, the use of the BOM is discouraged and +three bytes in the file. In UTF-8, the use of the BOM is discouraged and should generally be avoided. @@ -984,7 +984,7 @@ e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. .. impl-detail:: Some common encodings can bypass the codecs lookup machinery to - improve performance. These optimization opportunities are only + improve performance. These optimization opportunities are only recognized by CPython for a limited set of (case insensitive) aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and @@ -1145,7 +1145,7 @@ particular, the following variants typically exist: | iso2022_kr | csiso2022kr, iso2022kr, | Korean | | | iso-2022-kr | | +-----------------+--------------------------------+--------------------------------+ -| latin_1 | iso-8859-1, iso8859-1, 8859, | West Europe | +| latin_1 | iso-8859-1, iso8859-1, 8859, | Western Europe | | | cp819, latin, latin1, L1 | | +-----------------+--------------------------------+--------------------------------+ | iso8859_2 | iso-8859-2, latin2, L2 | Central and Eastern Europe | @@ -1249,11 +1249,11 @@ Python Specific Encodings ------------------------- A number of predefined codecs are specific to Python, so their codec names have -no meaning outside Python. These are listed in the tables below based on the +no meaning outside Python. These are listed in the tables below based on the expected input and output types (note that while text encodings are the most common use case for codecs, the underlying codec infrastructure supports -arbitrary data transforms rather than just text encodings). For asymmetric -codecs, the stated purpose describes the encoding direction. +arbitrary data transforms rather than just text encodings). For asymmetric +codecs, the stated meaning describes the encoding direction. Text Encodings ^^^^^^^^^^^^^^ @@ -1265,27 +1265,27 @@ encodings. .. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| +--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | +| Codec | Aliases | Meaning | +====================+=========+===========================+ -| idna | | Implements :rfc:`3490`, | +| idna | | Implement :rfc:`3490`, | | | | see also | | | | :mod:`encodings.idna`. | | | | Only ``errors='strict'`` | | | | is supported. | +--------------------+---------+---------------------------+ -| mbcs | ansi, | Windows only: Encode | +| mbcs | ansi, | Windows only: Encode the | | | dbcs | operand according to the | -| | | ANSI codepage (CP_ACP) | +| | | ANSI codepage (CP_ACP). | +--------------------+---------+---------------------------+ -| oem | | Windows only: Encode | +| oem | | Windows only: Encode the | | | | operand according to the | -| | | OEM codepage (CP_OEMCP) | +| | | OEM codepage (CP_OEMCP). | | | | | | | | .. versionadded:: 3.6 | +--------------------+---------+---------------------------+ -| palmos | | Encoding of PalmOS 3.5 | +| palmos | | Encoding of PalmOS 3.5. | +--------------------+---------+---------------------------+ -| punycode | | Implements :rfc:`3492`. | +| punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +--------------------+---------+---------------------------+ @@ -1308,8 +1308,8 @@ encodings. | | | literal in ASCII-encoded | | | | Python source code, | | | | except that quotes are | -| | | not escaped. Decodes from | -| | | Latin-1 source code. | +| | | not escaped. Decode | +| | | from Latin-1 source code. | | | | Beware that Python source | | | | code actually uses UTF-8 | | | | by default. | @@ -1325,19 +1325,19 @@ Binary Transforms ^^^^^^^^^^^^^^^^^ The following codecs provide binary transforms: :term:`bytes-like object` -to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` +to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` (which only produces :class:`str` output). .. tabularcolumns:: |l|L|L|L| +----------------------+------------------+------------------------------+------------------------------+ -| Codec | Aliases | Purpose | Encoder / decoder | +| Codec | Aliases | Meaning | Encoder / decoder | +======================+==================+==============================+==============================+ -| base64_codec [#b64]_ | base64, base_64 | Convert operand to multiline | :meth:`base64.encodebytes` / | -| | | MIME base64 (the result | :meth:`base64.decodebytes` | -| | | always includes a trailing | | -| | | ``'\n'``) | | +| base64_codec [#b64]_ | base64, base_64 | Convert the operand to | :meth:`base64.encodebytes` / | +| | | multiline MIME base64 (the | :meth:`base64.decodebytes` | +| | | result always includes a | | +| | | trailing ``'\n'``). | | | | | | | | | | .. versionchanged:: 3.4 | | | | | accepts any | | @@ -1345,23 +1345,23 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` | | | as input for encoding and | | | | | decoding | | +----------------------+------------------+------------------------------+------------------------------+ -| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / | -| | | using bz2 | :meth:`bz2.decompress` | +| bz2_codec | bz2 | Compress the operand using | :meth:`bz2.compress` / | +| | | bz2. | :meth:`bz2.decompress` | +----------------------+------------------+------------------------------+------------------------------+ -| hex_codec | hex | Convert operand to | :meth:`binascii.b2a_hex` / | +| hex_codec | hex | Convert the operand to | :meth:`binascii.b2a_hex` / | | | | hexadecimal | :meth:`binascii.a2b_hex` | | | | representation, with two | | -| | | digits per byte | | +| | | digits per byte. | | +----------------------+------------------+------------------------------+------------------------------+ -| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encode` with | -| | quotedprintable, | quoted printable | ``quotetabs=True`` / | +| quopri_codec | quopri, | Convert the operand to MIME | :meth:`quopri.encode` with | +| | quotedprintable, | quoted printable. | ``quotetabs=True`` / | | | quoted_printable | | :meth:`quopri.decode` | +----------------------+------------------+------------------------------+------------------------------+ | uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode | :meth:`uu.decode` | +| | | uuencode. | :meth:`uu.decode` | +----------------------+------------------+------------------------------+------------------------------+ -| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / | -| | | using gzip | :meth:`zlib.decompress` | +| zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | +| | | gzip. | :meth:`zlib.decompress` | +----------------------+------------------+------------------------------+------------------------------+ .. [#b64] In addition to :term:`bytes-like objects `, @@ -1381,16 +1381,17 @@ Text Transforms ^^^^^^^^^^^^^^^ The following codec provides a text transform: a :class:`str` to :class:`str` -mapping. It is not supported by :meth:`str.encode` (which only produces +mapping. It is not supported by :meth:`str.encode` (which only produces :class:`bytes` output). .. tabularcolumns:: |l|l|L| +--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | +| Codec | Aliases | Meaning | +====================+=========+===========================+ -| rot_13 | rot13 | Returns the Caesar-cypher | -| | | encryption of the operand | +| rot_13 | rot13 | Return the Caesar-cypher | +| | | encryption of the | +| | | operand. | +--------------------+---------+---------------------------+ .. versionadded:: 3.2 @@ -1412,6 +1413,9 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. +If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the +third-party `idna module _`. + These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as ``www.Alliancefrançaise.nu``) is converted into an ASCII-compatible encoding @@ -1428,7 +1432,7 @@ conversion between Unicode and ACE, separating an input string into labels based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>` and converting each label to ACE as required, and conversely separating an input byte string into labels based on the ``.`` separator and converting any ACE -labels found into unicode. Furthermore, the :mod:`socket` module +labels found into unicode. Furthermore, the :mod:`socket` module transparently converts Unicode host names to ACE, so that applications need not be concerned about converting host names themselves when they pass them to the socket module. On top of that, modules that have host names as function @@ -1437,7 +1441,7 @@ names (:mod:`http.client` then also transparently sends an IDNA hostname in the :mailheader:`Host` field if it sends that field at all). When receiving host names from the wire (such as in reverse name lookup), no -automatic conversion to Unicode is performed: Applications wishing to present +automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. The module :mod:`encodings.idna` also implements the nameprep procedure, which @@ -1469,7 +1473,7 @@ functions can be used directly if desired. .. module:: encodings.mbcs :synopsis: Windows ANSI codepage -Encode operand according to the ANSI codepage (CP_ACP). +This module implements the ANSI codepage (CP_ACP). .. availability:: Windows only. @@ -1488,7 +1492,7 @@ Encode operand according to the ANSI codepage (CP_ACP). :synopsis: UTF-8 codec with BOM signature .. moduleauthor:: Walter Dörwald -This module implements a variant of the UTF-8 codec: On encoding a UTF-8 encoded +This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this -is only done once (on the first write to the byte stream). For decoding an +is only done once (on the first write to the byte stream). On decoding, an optional UTF-8 encoded BOM at the start of the data will be skipped. diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index a52d2c62c4fea1..c66b9d3ec0a26d 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -43,8 +43,9 @@ To do just the former: :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. The *symbol* argument determines whether *source* is compiled as a statement - (``'single'``, the default) or as an :term:`expression` (``'eval'``). Any - other value will cause :exc:`ValueError` to be raised. + (``'single'``, the default), as a sequence of statements (``'exec'``) or + as an :term:`expression` (``'eval'``). Any other value will + cause :exc:`ValueError` to be raised. .. note:: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 2a3fb142f7297e..db0e25bb0772eb 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -98,12 +98,20 @@ ABC Inherits from Abstract Methods Mixin .. class:: Container - Hashable - Sized - Callable - ABCs for classes that provide respectively the methods :meth:`__contains__`, - :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`. + ABC for classes that provide the :meth:`__contains__` method. + +.. class:: Hashable + + ABC for classes that provide the :meth:`__hash__` method. + +.. class:: Sized + + ABC for classes that provide the :meth:`__len__` method. + +.. class:: Callable + + ABC for classes that provide the :meth:`__call__` method. .. class:: Iterable @@ -185,7 +193,7 @@ ABC Inherits from Abstract Methods Mixin expressions. Custom implementations must provide the :meth:`__await__` method. - :term:`Coroutine` objects and instances of the + :term:`Coroutine ` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 90a3f4bea9a45b..7cbc1721df151f 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -33,10 +33,10 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :class:`UserString` wrapper around string objects for easier string subclassing ===================== ==================================================================== -.. deprecated-removed:: 3.3 3.9 +.. deprecated-removed:: 3.3 3.10 Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. For backwards compatibility, they continue to be visible in this module through - Python 3.8. + Python 3.9. :class:`ChainMap` objects @@ -125,7 +125,7 @@ The class can be used to simulate nested scopes and is useful in templating. writing to any mapping in the chain. * Django's `Context class - `_ + `_ for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the @@ -162,7 +162,7 @@ environment variables which in turn take precedence over default values:: parser.add_argument('-u', '--user') parser.add_argument('-c', '--color') namespace = parser.parse_args() - command_line_args = {k:v for k, v in vars(namespace).items() if v} + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} combined = ChainMap(command_line_args, os.environ, defaults) print(combined['color']) @@ -1150,6 +1150,8 @@ variants of :func:`functools.lru_cache`:: return value def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) super().__setitem__(key, value) if len(self) > self.maxsize: oldest = next(iter(self)) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index bb5000a0736ccc..9ce5ca819c68a1 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -120,7 +120,7 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -185,10 +185,13 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + .. versionchanged:: 3.8 Setting *workers* to 0 now chooses the optimal number of cores. -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. @@ -232,7 +235,10 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) Byte-compile all the :file:`.py` files found along ``sys.path``. Return a true value if all the files compiled successfully, and a false value otherwise. @@ -255,6 +261,9 @@ Public functions .. versionchanged:: 3.7 The *invalidation_mode* parameter was added. + .. versionchanged:: 3.7.2 + The *invalidation_mode* parameter's default value is updated to None. + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 39cd9ff4826597..9813e9344f93a9 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -21,6 +21,7 @@ multitasking). Here's an overview: subprocess.rst sched.rst queue.rst + contextvars.rst The following are support modules for some of the above services: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index f2491dd24571be..cf9cd4c1ecc02d 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -182,7 +182,7 @@ ThreadPoolExecutor Example 'http://www.cnn.com/', 'http://europe.wsj.com/', 'http://www.bbc.co.uk/', - 'http://some-made-up-domain.com/'] + 'http://nonexistant-subdomain.python.org/'] # Retrieve a single page and report the URL and contents def load_url(url, timeout): @@ -209,7 +209,8 @@ ProcessPoolExecutor The :class:`ProcessPoolExecutor` class is an :class:`Executor` subclass that uses a pool of processes to execute calls asynchronously. :class:`ProcessPoolExecutor` uses the :mod:`multiprocessing` module, which -allows it to side-step the :term:`Global Interpreter Lock` but also means that +allows it to side-step the :term:`Global Interpreter Lock +` but also means that only picklable objects can be executed and returned. The ``__main__`` module must be importable by worker subprocesses. This means @@ -223,9 +224,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` + If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. - On Windows, *max_workers* must be equal or lower than ``61``. If it is not + On Windows, *max_workers* must be less than or equal to ``61``. If it is not then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then the default chosen will be at most ``61``, even if more processors are available. @@ -237,7 +238,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. each worker process; *initargs* is a tuple of arguments passed to the initializer. Should *initializer* raise an exception, all currently pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, - as well any attempt to submit more jobs to the pool. + as well as any attempt to submit more jobs to the pool. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a @@ -306,9 +307,10 @@ The :class:`Future` class encapsulates the asynchronous execution of a callable. .. method:: cancel() - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. + Attempt to cancel the call. If the call is currently being executed or + finished running and cannot be cancelled then the method will return + ``False``, otherwise the call will be cancelled and the method will + return ``True``. .. method:: cancelled() @@ -423,8 +425,9 @@ Module Functions Wait for the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* to complete. Returns a named 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. + completed (finished or cancelled futures) before the wait completed. The + second set, named ``not_done``, contains the futures that did not complete + (pending or running futures). *timeout* can be used to control the maximum number of seconds to wait before returning. *timeout* can be an int or float. If *timeout* is not specified @@ -455,7 +458,7 @@ Module Functions Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that + they complete (finished or cancelled futures). Any futures given by *fs* that are duplicated will be returned once. Any futures that completed before :func:`as_completed` is called will be yielded first. The returned iterator raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 04b52dc7b21531..2e22a549ee2813 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -313,6 +313,8 @@ from ``get()`` calls. my_dir: %(home_dir)s/lumberjack my_pictures: %(my_dir)s/Pictures + [Escape] + gain: 80%% # use a %% to escape the % sign (% is the only character that needs to be escaped) In the example above, :class:`ConfigParser` with *interpolation* set to ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of @@ -346,6 +348,9 @@ from ``get()`` calls. my_dir: ${home_dir}/lumberjack my_pictures: ${my_dir}/Pictures + [Escape] + cost: $$80 # use a $$ to escape the $ sign ($ is the only character that needs to be escaped) + Values from other sections can be fetched as well: .. code-block:: ini @@ -669,97 +674,98 @@ be overridden by subclasses or by attribute assignment. .. attribute:: ConfigParser.BOOLEAN_STATES - By default when using :meth:`~ConfigParser.getboolean`, config parsers - consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, - ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, - ``'off'``. You can override this by specifying a custom dictionary of strings - and their Boolean outcomes. For example: + By default when using :meth:`~ConfigParser.getboolean`, config parsers + consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, + ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, + ``'off'``. You can override this by specifying a custom dictionary of strings + and their Boolean outcomes. For example: - .. doctest:: + .. doctest:: - >>> custom = configparser.ConfigParser() - >>> custom['section1'] = {'funky': 'nope'} - >>> custom['section1'].getboolean('funky') - Traceback (most recent call last): - ... - ValueError: Not a boolean: nope - >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} - >>> custom['section1'].getboolean('funky') - False + >>> custom = configparser.ConfigParser() + >>> custom['section1'] = {'funky': 'nope'} + >>> custom['section1'].getboolean('funky') + Traceback (most recent call last): + ... + ValueError: Not a boolean: nope + >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} + >>> custom['section1'].getboolean('funky') + False - Other typical Boolean pairs include ``accept``/``reject`` or - ``enabled``/``disabled``. + Other typical Boolean pairs include ``accept``/``reject`` or + ``enabled``/``disabled``. .. method:: ConfigParser.optionxform(option) + :noindex: - This method transforms option names on every read, get, or set - operation. The default converts the name to lowercase. This also - means that when a configuration file gets written, all keys will be - lowercase. Override this method if that's unsuitable. - For example: + This method transforms option names on every read, get, or set + operation. The default converts the name to lowercase. This also + means that when a configuration file gets written, all keys will be + lowercase. Override this method if that's unsuitable. + For example: - .. doctest:: + .. doctest:: + + >>> config = """ + ... [Section1] + ... Key = Value + ... + ... [Section2] + ... AnotherKey = Value + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> list(typical['Section1'].keys()) + ['key'] + >>> list(typical['Section2'].keys()) + ['anotherkey'] + >>> custom = configparser.RawConfigParser() + >>> custom.optionxform = lambda option: option + >>> custom.read_string(config) + >>> list(custom['Section1'].keys()) + ['Key'] + >>> list(custom['Section2'].keys()) + ['AnotherKey'] - >>> config = """ - ... [Section1] - ... Key = Value - ... - ... [Section2] - ... AnotherKey = Value - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> list(typical['Section1'].keys()) - ['key'] - >>> list(typical['Section2'].keys()) - ['anotherkey'] - >>> custom = configparser.RawConfigParser() - >>> custom.optionxform = lambda option: option - >>> custom.read_string(config) - >>> list(custom['Section1'].keys()) - ['Key'] - >>> list(custom['Section2'].keys()) - ['AnotherKey'] - - .. note:: - The optionxform function transforms option names to a canonical form. - This should be an idempotent function: if the name is already in - canonical form, it should be returned unchanged. + .. note:: + The optionxform function transforms option names to a canonical form. + This should be an idempotent function: if the name is already in + canonical form, it should be returned unchanged. .. attribute:: ConfigParser.SECTCRE - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: + A compiled regular expression used to parse section headers. The default + matches ``[section]`` to the name ``"section"``. Whitespace is considered + part of the section name, thus ``[ larch ]`` will be read as a section of + name ``" larch "``. Override this attribute if that's unsuitable. For + example: + + .. doctest:: + + >>> import re + >>> config = """ + ... [Section 1] + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> typical.sections() + ['Section 1', ' Section 2 '] + >>> custom = configparser.ConfigParser() + >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") + >>> custom.read_string(config) + >>> custom.sections() + ['Section 1', 'Section 2'] - .. doctest:: + .. note:: - >>> import re - >>> config = """ - ... [Section 1] - ... option = value - ... - ... [ Section 2 ] - ... another = val - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> typical.sections() - ['Section 1', ' Section 2 '] - >>> custom = configparser.ConfigParser() - >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") - >>> custom.read_string(config) - >>> custom.sections() - ['Section 1', 'Section 2'] - - .. note:: - - While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing - option lines, it's not recommended to override it because that would - interfere with constructor options *allow_no_value* and *delimiters*. + While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing + option lines, it's not recommended to override it because that would + interfere with constructor options *allow_no_value* and *delimiters*. Legacy API Examples diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 73b24e5f251a94..6b2286e8968eb0 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -638,7 +638,7 @@ even further by means of a small helper class:: class Callback(ExitStack): def __init__(self, callback, /, *args, **kwds): - super(Callback, self).__init__() + super().__init__() self.callback(callback, *args, **kwds) def cancel(self): diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 8805661c456edb..be1dd0c9eb57e8 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -26,7 +26,7 @@ See also :pep:`567` for additional details. Context Variables ----------------- -.. class:: ContextVar(name, [\*, default]) +.. class:: ContextVar(name, [*, default]) This class is used to declare a new Context Variable, e.g.:: @@ -94,7 +94,7 @@ Context Variables # var.get() would raise a LookupError. -.. class:: contextvars.Token +.. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert @@ -146,7 +146,7 @@ Manual Context Management Context implements the :class:`collections.abc.Mapping` interface. - .. method:: run(callable, \*args, \*\*kwargs) + .. method:: run(callable, *args, **kwargs) Execute ``callable(*args, **kwargs)`` code in the context object the *run* method is called on. Return the result of the execution diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index c7bd89f9637226..0eb5a793ad953a 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -27,10 +27,11 @@ Interface summary: Return a deep copy of *x*. -.. exception:: error +.. exception:: Error Raised for module specific errors. +.. _shallow_vs_deep_copy: The difference between shallow and deep copying is only relevant for compound objects (objects that contain other objects, like lists or class instances): diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 49e22fa73ed265..7a72c26d5badeb 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -161,11 +161,15 @@ The :mod:`csv` module defines the following classes: If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults to ``None``). If a non-blank row has fewer fields than fieldnames, the - missing values are filled-in with ``None``. + missing values are filled-in with the value of *restval* (which defaults + to ``None``). All other optional or keyword arguments are passed to the underlying :class:`reader` instance. + .. versionchanged:: 3.6 + Returned rows are now of type :class:`OrderedDict`. + .. versionchanged:: 3.8 Returned rows are now of type :class:`dict`. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 97172c588ae97f..fd6422cc8c06c5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -152,8 +152,8 @@ the ``time()`` function, which returns system time in seconds since the Unix epoch, and the ``GetModuleHandleA()`` function, which returns a win32 module handle. -This example calls both functions with a NULL pointer (``None`` should be used -as the NULL pointer):: +This example calls both functions with a ``NULL`` pointer (``None`` should be used +as the ``NULL`` pointer):: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 @@ -161,13 +161,6 @@ as the NULL pointer):: 0x1d000000 >>> -.. note:: - - :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if - it detects that an invalid number of arguments were passed. This behavior - should not be relied upon. It is deprecated in 3.6.2, and will be removed - in 3.7. - :exc:`ValueError` is raised when you call an ``stdcall`` function with the ``cdecl`` calling convention, or vice versa:: @@ -576,7 +569,7 @@ Here is a simple example of a POINT structure, which contains two integers named >>> POINT(1, 2, 3) Traceback (most recent call last): File "", line 1, in - ValueError: too many initializers + TypeError: too many initializers >>> You can, however, build much more complicated structures. A structure can @@ -624,7 +617,7 @@ Structure/union alignment and byte order ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior be specifying a +compiler does it. It is possible to override this behavior by specifying a :attr:`_pack_` class attribute in the subclass definition. This must be set to a positive integer and specifies the maximum alignment for the fields. This is what ``#pragma pack(n)`` also does in MSVC. @@ -922,13 +915,13 @@ attribute later, after the class statement:: ... ("next", POINTER(cell))] >>> -Lets try it. We create two instances of ``cell``, and let them point to each +Let's try it. We create two instances of ``cell``, and let them point to each other, and finally follow the pointer chain a few times:: >>> c1 = cell() - >>> c1.name = "foo" + >>> c1.name = b"foo" >>> c2 = cell() - >>> c2.name = "bar" + >>> c2.name = b"bar" >>> c1.next = pointer(c2) >>> c2.next = pointer(c1) >>> p = c1 @@ -1083,7 +1076,7 @@ An extended example which also demonstrates the use of pointers accesses the Quoting the docs for that value: This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen + records, terminated by one whose members are all ``NULL`` or zero. When a frozen module is imported, it is searched in this table. Third-party code could play tricks with this to provide a dynamically created collection of frozen modules. @@ -1110,7 +1103,7 @@ Since ``table`` is a ``pointer`` to the array of ``struct_frozen`` records, we can iterate over it, but we just have to make sure that our loop terminates, because pointers have no size. Sooner or later it would probably crash with an access violation or whatever, so it's better to break out of the loop when we -hit the NULL entry:: +hit the ``NULL`` entry:: >>> for item in table: ... if item.name is None: @@ -1125,8 +1118,8 @@ hit the NULL entry:: >>> The fact that standard Python has a frozen module and a frozen package -(indicated by the negative size member) is not well known, it is only used for -testing. Try it out with ``import __hello__`` for example. +(indicated by the negative ``size`` member) is not well known, it is only used +for testing. Try it out with ``import __hello__`` for example. .. _ctypes-surprises: @@ -1175,16 +1168,21 @@ Keep in mind that retrieving sub-objects from Structure, Unions, and Arrays doesn't *copy* the sub-object, instead it retrieves a wrapper object accessing the root-object's underlying buffer. -Another example that may behave different from what one would expect is this:: +Another example that may behave differently from what one would expect is this:: >>> s = c_char_p() - >>> s.value = "abc def ghi" + >>> s.value = b"abc def ghi" >>> s.value - 'abc def ghi' + b'abc def ghi' >>> s.value is s.value False >>> +.. note:: + + Objects instantiated from :class:`c_char_p` can only have their value set to bytes + or integers. + Why is it printing ``False``? ctypes instances are objects containing a memory block plus some :term:`descriptor`\s accessing the contents of the memory. Storing a Python object in the memory block does not store the object itself, @@ -1328,6 +1326,21 @@ way is to instantiate one of the following classes: libraries use the standard C calling convention, and are assumed to return :c:type:`int`. + On Windows creating a :class:`CDLL` instance may fail even if the DLL name + exists. When a dependent DLL of the loaded DLL is not found, a + :exc:`OSError` error is raised with the message *"[WinError 126] The + specified module could not be found".* This error message does not contain + the name of the missing DLL because the Windows API does not return this + information making this error hard to diagnose. To resolve this error and + determine which DLL is not found, you need to find the list of dependent + DLLs and determine which one is not found using Windows debugging and + tracing tools. + +.. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + .. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=0) @@ -1509,18 +1522,24 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. -.. audit-event:: ctypes.dlopen name +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader Loading a library through any of these objects raises an :ref:`auditing event ` ``ctypes.dlopen`` with string argument ``name``, the name used to load the library. -.. audit-event:: ctypes.dlsym "library name" +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader Accessing a function on a loaded library raises an auditing event ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` (the symbol's name as a string or integer). +.. audit-event:: ctypes.dlsym/handle handle,name ctypes.LibraryLoader + + In cases when only the library handle is available rather than the object, + accessing a function raises an auditing event ``ctypes.dlsym/handle`` with + arguments ``handle`` (the raw library handle) and ``name``. + .. _ctypes-foreign-functions: Foreign functions @@ -1606,6 +1625,19 @@ They are instances of a private class: passed arguments. +.. audit-event:: ctypes.seh_exception code foreign-functions + + On Windows, when a foreign function call raises a system exception (for + example, due to an access violation), it will be captured and replaced with + a suitable Python exception. Further, an auditing event + ``ctypes.seh_exception`` with argument ``code`` will be raised, allowing an + audit hook to replace the exception with its own. + +.. audit-event:: ctypes.call_function func_pointer,arguments foreign-functions + + Some ways to invoke foreign function calls may raise an auditing event + ``ctypes.call_function`` with arguments ``function pointer`` and ``arguments``. + .. _ctypes-function-prototypes: Function prototypes @@ -1797,6 +1829,8 @@ Utility functions Returns the address of the memory buffer as integer. *obj* must be an instance of a ctypes type. + .. audit-event:: ctypes.addressof obj ctypes.addressof + .. function:: alignment(obj_or_type) @@ -1839,6 +1873,7 @@ Utility functions termination character. An integer can be passed as second argument which allows specifying the size of the array if the length of the bytes should not be used. + .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer .. function:: create_unicode_buffer(init_or_size, size=None) @@ -1855,6 +1890,7 @@ Utility functions allows specifying the size of the array if the length of the string should not be used. + .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer .. function:: DllCanUnloadNow() @@ -1912,11 +1948,15 @@ Utility functions Returns the current value of the ctypes-private copy of the system :data:`errno` variable in the calling thread. + .. audit-event:: ctypes.get_errno "" ctypes.get_errno + .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system :data:`LastError` variable in the calling thread. + .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error + .. function:: memmove(dst, src, count) Same as the standard C memmove library function: copies *count* bytes from @@ -1960,6 +2000,7 @@ Utility functions Set the current value of the ctypes-private copy of the system :data:`errno` variable in the calling thread to *value* and return the previous value. + .. audit-event:: ctypes.set_errno errno ctypes.set_errno .. function:: set_last_error(value) @@ -1968,6 +2009,7 @@ Utility functions :data:`LastError` variable in the calling thread to *value* and return the previous value. + .. audit-event:: ctypes.set_last_error error ctypes.set_last_error .. function:: sizeof(obj_or_type) @@ -1982,6 +2024,8 @@ Utility functions object. If size is specified, it is used as size, otherwise the string is assumed to be zero-terminated. + .. audit-event:: ctypes.string_at address,size ctypes.string_at + .. function:: WinError(code=None, descr=None) @@ -2002,6 +2046,8 @@ Utility functions characters of the string, otherwise the string is assumed to be zero-terminated. + .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at + .. _ctypes-data-types: @@ -2029,6 +2075,7 @@ Data types source buffer in bytes; the default is zero. If the source buffer is not large enough a :exc:`ValueError` is raised. + .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer .. method:: _CData.from_buffer_copy(source[, offset]) @@ -2038,15 +2085,17 @@ Data types is zero. If the source buffer is not large enough a :exc:`ValueError` is raised. + .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer_copy + .. method:: from_address(address) This method returns a ctypes type instance using the memory specified by *address* which must be an integer. - .. audit-event:: ctypes.cdata address + .. audit-event:: ctypes.cdata address ctypes._CData.from_address This method, and others that indirectly call this method, raises an - :func:`auditing event ` ``ctypes.cdata`` with argument + :ref:`auditing event ` ``ctypes.cdata`` with argument ``address``. .. method:: from_param(obj) @@ -2459,7 +2508,7 @@ other data types containing pointer type fields. Arrays and pointers ^^^^^^^^^^^^^^^^^^^ -.. class:: Array(\*args) +.. class:: Array(*args) Abstract base class for arrays. @@ -2511,4 +2560,3 @@ Arrays and pointers Returns the object to which to pointer points. Assigning to this attribute changes the pointer to point to the assigned object. - diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 7d1e7538a292b3..5a7536412e6ce6 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -112,14 +112,15 @@ The module :mod:`curses` defines the following functions: .. function:: color_content(color_number) Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, + *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between ``0`` (no component) and ``1000`` (maximum amount of component). -.. function:: color_pair(color_number) +.. function:: color_pair(pair_number) - Return the attribute value for displaying text in the specified color. This + Return the attribute value for displaying text in the specified color pair. + Only the first 256 color pairs are supported. This attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart to this function. @@ -214,7 +215,7 @@ The module :mod:`curses` defines the following functions: .. function:: getmouse() After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be call to retrieve the queued mouse event, represented as a + method should be called to retrieve the queued mouse event, represented as a 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is currently unused.) *bstate* is an integer value whose bits will be set to @@ -278,7 +279,7 @@ The module :mod:`curses` defines the following functions: Change the definition of a color, taking the number of the color to be changed followed by three RGB values (for the amounts of red, green, and blue components). The value of *color_number* must be between ``0`` and - :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and + `COLORS - 1`. Each of *r*, *g*, *b*, must be a value between ``0`` and ``1000``. When :func:`init_color` is used, all occurrences of that color on the screen immediately change to the new definition. This function is a no-op on most terminals; it is active only if :func:`can_change_color` returns ``True``. @@ -291,7 +292,8 @@ The module :mod:`curses` defines the following functions: color number. The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot be changed). The value of *fg* and *bg* arguments must be between ``0`` and - :const:`COLORS`. If the color-pair was previously initialized, the screen is + ``COLORS - 1``, or, after calling :func:`use_default_colors`, ``-1``. + If the color-pair was previously initialized, the screen is refreshed and all occurrences of that color-pair are changed to the new definition. @@ -441,7 +443,7 @@ The module :mod:`curses` defines the following functions: .. function:: pair_content(pair_number) Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. - The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1``. + The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. .. function:: pair_number(attr) @@ -682,7 +684,7 @@ the following methods and attributes: window.addch(y, x, ch[, attr]) Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painter at that location. By default, the character + character previously painted at that location. By default, the character position and attributes are the current settings for the window object. .. note:: @@ -1283,7 +1285,7 @@ The :mod:`curses` module defines the following data members: .. data:: ERR - Some curses routines that return an integer, such as :func:`getch`, return + Some curses routines that return an integer, such as :meth:`~window.getch`, return :const:`ERR` upon failure. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index db5c3e0c7e2893..c47ee0a932ae24 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -19,9 +19,11 @@ in :pep:`557`. The member variables to use in these generated methods are defined using :pep:`526` type annotations. For example this code:: + from dataclasses import dataclass + @dataclass class InventoryItem: - '''Class for keeping track of an item in inventory.''' + """Class for keeping track of an item in inventory.""" name: str unit_price: float quantity_on_hand: int = 0 @@ -31,7 +33,7 @@ using :pep:`526` type annotations. For example this code:: Will add, among other things, a :meth:`__init__` that looks like:: - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name self.unit_price = unit_price self.quantity_on_hand = quantity_on_hand @@ -60,8 +62,9 @@ Module-level decorators, classes, and functions The :func:`dataclass` decorator will add various "dunder" methods to the class, described below. If any of the added methods already - exist on the class, a :exc:`TypeError` will be raised. The decorator - returns the same class that is called on: no new class is created. + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. If :func:`dataclass` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this @@ -115,7 +118,7 @@ Module-level decorators, classes, and functions If the class already defines any of :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`ValueError` is raised. + :exc:`TypeError` is raised. - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method is generated according to how ``eq`` and ``frozen`` are set. @@ -133,7 +136,7 @@ Module-level decorators, classes, and functions attribute ``__hash__ = None`` has a specific meaning to Python, as described in the :meth:`__hash__` documentation. - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + If :meth:`__hash__` is not explicitly defined, or if it is set to ``None``, then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. Although not recommended, you can force :func:`dataclass` to create a :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case @@ -153,7 +156,7 @@ Module-level decorators, classes, and functions method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - - ``frozen``: If true (the default is False), assigning to fields will + - ``frozen``: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. @@ -386,8 +389,8 @@ Module-level decorators, classes, and functions .. function:: is_dataclass(class_or_instance) - Returns True if its parameter is a dataclass or an instance of one, - otherwise returns False. + Return ``True`` if its parameter is a dataclass or an instance of one, + otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -589,4 +592,4 @@ Exceptions Raised when an implicitly defined :meth:`__setattr__` or :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. + ``frozen=True``. It is a subclass of :exc:`AttributeError`. diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index 48af0823aa4912..94010c0e391b0b 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -5,13 +5,14 @@ Data Types ********** The modules described in this chapter provide a variety of specialized data -types such as dates and times, fixed-type arrays, heap queues, synchronized -queues, and sets. +types such as dates and times, fixed-type arrays, heap queues, double-ended +queues, and enumerations. Python also provides some built-in data types, in particular, :class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and :class:`tuple`. The :class:`str` class is used to hold -Unicode strings, and the :class:`bytes` class is used to hold binary data. +Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used +to hold binary data. The following modules are documented in this chapter: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index fb41aeeb5bed03..5ab3cc020c3f61 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -14,40 +14,60 @@ .. XXX what order should the types be discussed in? -The :mod:`datetime` module supplies classes for manipulating dates and times in -both simple and complex ways. While date and time arithmetic is supported, the -focus of the implementation is on efficient attribute extraction for output -formatting and manipulation. For related functionality, see also the -:mod:`time` and :mod:`calendar` modules. - -There are two kinds of date and time objects: "naive" and "aware". - -An aware object has sufficient knowledge of applicable algorithmic and -political time adjustments, such as time zone and daylight saving time -information, to locate itself relative to other aware objects. An aware object -is used to represent a specific moment in time that is not open to -interpretation [#]_. - -A naive object does not contain enough information to unambiguously locate -itself relative to other date/time objects. Whether a naive object represents +The :mod:`datetime` module supplies classes for manipulating dates and times. + +While date and time arithmetic is supported, the focus of the implementation is +on efficient attribute extraction for output formatting and manipulation. + +.. seealso:: + + Module :mod:`calendar` + General calendar related functions. + + Module :mod:`time` + Time access and conversions. + + Package `dateutil `_ + Third-party library with expanded time zone and parsing support. + +.. _datetime-naive-aware: + +Aware and Naive Objects +----------------------- + +Date and time objects may be categorized as "aware" or "naive" depending on +whether or not they include timezone information. + +With sufficient knowledge of applicable algorithmic and political time +adjustments, such as time zone and daylight saving time information, +an **aware** object can locate itself relative to other aware objects. +An aware object represents a specific moment in time that is not open to +interpretation. [#]_ + +A **naive** object does not contain enough information to unambiguously locate +itself relative to other date/time objects. Whether a naive object represents Coordinated Universal Time (UTC), local time, or time in some other timezone is purely up to the program, just like it is up to the program whether a -particular number represents metres, miles, or mass. Naive objects are easy to +particular number represents metres, miles, or mass. Naive objects are easy to understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that can be set to an instance of a subclass of the abstract :class:`tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC -time, the time zone name, and whether Daylight Saving Time is in effect. Note -that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offset from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of -detail is up to the application. The rules for time adjustment across the +time, the time zone name, and whether daylight saving time is in effect. + +Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is +supplied by the :mod:`datetime` module. The :class:`timezone` class can +represent simple timezones with fixed offsets from UTC, such as UTC itself or +North American EST and EDT timezones. Supporting timezones at deeper levels of +detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. +Constants +--------- + The :mod:`datetime` module exports the following constants: .. data:: MINYEAR @@ -61,16 +81,6 @@ The :mod:`datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is ``9999``. - -.. seealso:: - - Module :mod:`calendar` - General calendar related functions. - - Module :mod:`time` - Time access and conversions. - - Available Types --------------- @@ -86,7 +96,7 @@ Available Types :noindex: An idealized time, independent of any particular day, assuming that every day - has exactly 24\*60\*60 seconds (there is no notion of "leap seconds" here). + has exactly 24\*60\*60 seconds. (There is no notion of "leap seconds" here.) Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, and :attr:`.tzinfo`. @@ -109,7 +119,7 @@ Available Types .. class:: tzinfo :noindex: - An abstract base class for time zone information objects. These are used by the + An abstract base class for time zone information objects. These are used by the :class:`.datetime` and :class:`.time` classes to provide a customizable notion of time adjustment (for example, to account for time zone and/or daylight saving time). @@ -122,22 +132,8 @@ Available Types .. versionadded:: 3.2 - Objects of these types are immutable. -Objects of the :class:`date` type are always naive. - -An object of type :class:`.time` or :class:`.datetime` may be naive or aware. -A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and -``d.tzinfo.utcoffset(d)`` does not return ``None``. If ``d.tzinfo`` is -``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)`` -returns ``None``, *d* is naive. A :class:`.time` object *t* is aware -if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return -``None``. Otherwise, *t* is naive. - -The distinction between naive and aware doesn't apply to :class:`timedelta` -objects. - Subclass relationships:: object @@ -148,6 +144,40 @@ Subclass relationships:: date datetime +Common Properties +^^^^^^^^^^^^^^^^^ + +The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types +share these common features: + +- Objects of these types are immutable. +- Objects of these types are hashable, meaning that they can be used as + dictionary keys. +- Objects of these types support efficient pickling via the :mod:`pickle` module. + +Determining if an Object is Aware or Naive +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Objects of the :class:`date` type are always naive. + +An object of type :class:`.time` or :class:`.datetime` may be aware or naive. + +A :class:`.datetime` object *d* is aware if both of the following hold: + +1. ``d.tzinfo`` is not ``None`` +2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` + +Otherwise, *d* is naive. + +A :class:`.time` object *t* is aware if both of the following hold: + +1. ``t.tzinfo`` is not ``None`` +2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. + +Otherwise, *t* is naive. + +The distinction between aware and naive doesn't apply to :class:`timedelta` +objects. .. _datetime-timedelta: @@ -159,11 +189,11 @@ dates or times. .. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - All arguments are optional and default to ``0``. Arguments may be integers + All arguments are optional and default to ``0``. Arguments may be integers or floats, and may be positive or negative. - Only *days*, *seconds* and *microseconds* are stored internally. Arguments are - converted to those units: + Only *days*, *seconds* and *microseconds* are stored internally. + Arguments are converted to those units: * A millisecond is converted to 1000 microseconds. * A minute is converted to 60 seconds. @@ -177,10 +207,28 @@ dates or times. * ``0 <= seconds < 3600*24`` (the number of seconds in one day) * ``-999999999 <= days <= 999999999`` + The following example illustrates how any arguments besides + *days*, *seconds* and *microseconds* are "merged" and normalized into those + three resulting attributes:: + + >>> from datetime import timedelta + >>> delta = timedelta( + ... days=50, + ... seconds=27, + ... microseconds=10, + ... milliseconds=29000, + ... minutes=5, + ... hours=8, + ... weeks=2 + ... ) + >>> # Only days, seconds, and microseconds remain + >>> delta + datetime.timedelta(days=64, seconds=29156, microseconds=10) + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using - round-half-to-even tiebreaker. If no argument is a float, the + round-half-to-even tiebreaker. If no argument is a float, the conversion and normalization processes are exact (no information is lost). @@ -188,7 +236,7 @@ dates or times. :exc:`OverflowError` is raised. Note that normalization of negative values may be surprising at first. For - example, + example:: >>> from datetime import timedelta >>> d = timedelta(microseconds=-1) @@ -196,7 +244,7 @@ dates or times. (-1, 86399, 999999) -Class attributes are: +Class attributes: .. attribute:: timedelta.min @@ -263,7 +311,7 @@ Supported operations: | | timedelta.resolution using round-half-to-even.| +--------------------------------+-----------------------------------------------+ | ``t1 = t2 // i`` or | The floor is computed and the remainder (if | -| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | +| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | | | integer is returned. (3) | +--------------------------------+-----------------------------------------------+ | ``t1 = t2 % t3`` | The remainder is computed as a | @@ -282,8 +330,8 @@ Supported operations: | | -*t1.seconds*, -*t1.microseconds*), | | | and to *t1*\* -1. (1)(4) | +--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, and| -| | to -*t* when ``t.days < 0``. (2) | +| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | +| | and to -*t* when ``t.days < 0``. (2) | +--------------------------------+-----------------------------------------------+ | ``str(t)`` | Returns a string in the form | | | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | @@ -298,10 +346,10 @@ Supported operations: Notes: (1) - This is exact, but may overflow. + This is exact but may overflow. (2) - This is exact, and cannot overflow. + This is exact and cannot overflow. (3) Division by 0 raises :exc:`ZeroDivisionError`. @@ -310,41 +358,56 @@ Notes: -*timedelta.max* is not representable as a :class:`timedelta` object. (5) - String representations of :class:`timedelta` objects are normalized - similarly to their internal representation. This leads to somewhat - unusual results for negative timedeltas. For example: + String representations of :class:`timedelta` objects are normalized + similarly to their internal representation. This leads to somewhat + unusual results for negative timedeltas. For example:: - >>> timedelta(hours=-5) - datetime.timedelta(days=-1, seconds=68400) - >>> print(_) - -1 day, 19:00:00 + >>> timedelta(hours=-5) + datetime.timedelta(days=-1, seconds=68400) + >>> print(_) + -1 day, 19:00:00 (6) The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except when t3 is equal to ``timedelta.max``; in that case the former will produce a result while the latter will overflow. -In addition to the operations listed above :class:`timedelta` objects support +In addition to the operations listed above, :class:`timedelta` objects support certain additions and subtractions with :class:`date` and :class:`.datetime` objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another :class:`timedelta` object are now supported, as are remainder operations and - the :func:`divmod` function. True division and multiplication of a + the :func:`divmod` function. True division and multiplication of a :class:`timedelta` object by a :class:`float` object are now supported. -Comparisons of :class:`timedelta` objects are supported with the -:class:`timedelta` object representing the smaller duration considered to be the -smaller timedelta. In order to stop mixed-type comparisons from falling back to -the default comparison by object address, when a :class:`timedelta` object is -compared to an object of a different type, :exc:`TypeError` is raised unless the -comparison is ``==`` or ``!=``. The latter cases return :const:`False` or -:const:`True`, respectively. +Comparisons of :class:`timedelta` objects are supported, with some caveats. + +The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter +the type of the compared object:: + + >>> from datetime import timedelta + >>> delta1 = timedelta(seconds=57) + >>> delta2 = timedelta(hours=25, seconds=2) + >>> delta2 != delta1 + True + >>> delta2 == 5 + False + +For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` +object is compared to an object of a different type, :exc:`TypeError` +is raised:: + + >>> delta2 > delta1 + True + >>> delta2 > 5 + Traceback (most recent call last): + File "", line 1, in + TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' -:class:`timedelta` objects are :term:`hashable` (usable as dictionary keys), support -efficient pickling, and in Boolean contexts, a :class:`timedelta` object is +In Boolean contexts, a :class:`timedelta` object is considered to be true if and only if it isn't equal to ``timedelta(0)``. Instance methods: @@ -360,29 +423,36 @@ Instance methods: .. versionadded:: 3.2 +Examples of usage: :class:`timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Example usage: +An additional example of normalization:: + >>> # Components of another_year add up to exactly 365 days >>> from datetime import timedelta >>> year = timedelta(days=365) >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) # adds up to 365 days - >>> year.total_seconds() - 31536000.0 + ... minutes=50, seconds=600) >>> year == another_year True + >>> year.total_seconds() + 31536000.0 + +Examples of :class:`timedelta` arithmetic:: + + >>> from datetime import timedelta + >>> year = timedelta(days=365) >>> ten_years = 10 * year - >>> ten_years, ten_years.days // 365 - (datetime.timedelta(days=3650), 10) + >>> ten_years + datetime.timedelta(days=3650) + >>> ten_years.days // 365 + 10 >>> nine_years = ten_years - year - >>> nine_years, nine_years.days // 365 - (datetime.timedelta(days=3285), 9) + >>> nine_years + datetime.timedelta(days=3285) >>> three_years = nine_years // 3 >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) - >>> abs(three_years - ten_years) == 2 * three_years + year - True - .. _datetime-date: @@ -391,17 +461,14 @@ Example usage: A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both -directions. January 1 of year 1 is called day number 1, January 2 of year 1 is -called day number 2, and so on. This matches the definition of the "proleptic -Gregorian" calendar in Dershowitz and Reingold's book Calendrical Calculations, -where it's the base calendar for all computations. See the book for algorithms -for converting between proleptic Gregorian ordinals and many other calendar -systems. +directions. +January 1 of year 1 is called day number 1, January 2 of year 1 is +called day number 2, and so on. [#]_ .. class:: date(year, month, day) - All arguments are required. Arguments may be integers, in the following + All arguments are required. Arguments must be integers, in the following ranges: * ``MINYEAR <= year <= MAXYEAR`` @@ -415,17 +482,19 @@ Other constructors, all class methods: .. classmethod:: date.today() - Return the current local date. This is equivalent to - ``date.fromtimestamp(time.time())``. + Return the current local date. + This is equivalent to ``date.fromtimestamp(time.time())``. .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is returned - by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out - of the range of values supported by the platform C :c:func:`localtime` function, - and :exc:`OSError` on :c:func:`localtime` failure. - It's common for this to be restricted to years from 1970 through 2038. Note + Return the local date corresponding to the POSIX timestamp, such as is + returned by :func:`time.time`. + + This may raise :exc:`OverflowError`, if the timestamp is out + of the range of values supported by the platform C :c:func:`localtime` + function, and :exc:`OSError` on :c:func:`localtime` failure. + It's common for this to be restricted to years from 1970 through 2038. Note that on non-POSIX systems that include leap seconds in their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`. @@ -438,24 +507,27 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where January - 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, ``date.fromordinal(d.toordinal()) == - d``. + Return the date corresponding to the proleptic Gregorian ordinal, where + January 1 of year 1 has ordinal 1. + + :exc:`ValueError` is raised unless ``1 <= ordinal <= + date.max.toordinal()``. For any date *d*, + ``date.fromordinal(d.toordinal()) == d``. .. classmethod:: date.fromisoformat(date_string) - Return a :class:`date` corresponding to a *date_string* in the format emitted - by :meth:`date.isoformat`. Specifically, this function supports strings in - the format(s) ``YYYY-MM-DD``. + Return a :class:`date` corresponding to a *date_string* given in the format + ``YYYY-MM-DD``:: - .. caution:: + >>> from datetime import date + >>> date.fromisoformat('2019-12-04') + datetime.date(2019, 12, 4) - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`date.isoformat`. + This is the inverse of :meth:`date.isoformat`. It only supports the format + ``YYYY-MM-DD``. - .. versionadded:: 3.7 + .. versionadded:: 3.7 .. classmethod:: date.fromisocalendar(year, week, day) @@ -507,7 +579,7 @@ Supported operations: | Operation | Result | +===============================+==============================================+ | ``date2 = date1 + timedelta`` | *date2* is ``timedelta.days`` days removed | -| | from *date1*. (1) | +| | from *date1*. (1) | +-------------------------------+----------------------------------------------+ | ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | | | timedelta == date1``. (2) | @@ -522,7 +594,7 @@ Notes: (1) *date2* is moved forward in time if ``timedelta.days > 0``, or backward if - ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. + ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. :exc:`OverflowError` is raised if ``date2.year`` would be smaller than :const:`MINYEAR` or larger than :const:`MAXYEAR`. @@ -531,7 +603,7 @@ Notes: ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. (3) - This is exact, and cannot overflow. timedelta.seconds and + This is exact, and cannot overflow. timedelta.seconds and timedelta.microseconds are 0, and date2 + timedelta == date1 after. (4) @@ -539,38 +611,47 @@ Notes: date2.toordinal()``. Date comparison raises :exc:`TypeError` if the other comparand isn't also a :class:`date` object. However, ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a + :meth:`timetuple` attribute. This hook gives other kinds of date objects a chance at implementing mixed-type comparison. If not, when a :class:`date` object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return + unless the comparison is ``==`` or ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -Dates can be used as dictionary keys. In Boolean contexts, all :class:`date` -objects are considered to be true. +In Boolean contexts, all :class:`date` objects are considered to be true. Instance methods: .. method:: date.replace(year=self.year, month=self.month, day=self.day) Return a date with the same value, except for those parameters given new - values by whichever keyword arguments are specified. For example, if ``d == - date(2002, 12, 31)``, then ``d.replace(day=26) == date(2002, 12, 26)``. + values by whichever keyword arguments are specified. + + Example:: + + >>> from datetime import date + >>> d = date(2002, 12, 31) + >>> d.replace(day=26) + datetime.date(2002, 12, 26) .. method:: date.timetuple() Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - The hours, minutes and seconds are 0, and the DST flag is -1. ``d.timetuple()`` - is equivalent to ``time.struct_time((d.year, d.month, d.day, 0, 0, 0, - d.weekday(), yday, -1))``, where ``yday = d.toordinal() - date(d.year, 1, - 1).toordinal() + 1`` is the day number within the current year starting with - ``1`` for January 1st. + + The hours, minutes and seconds are 0, and the DST flag is -1. + + ``d.timetuple()`` is equivalent to:: + + time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) + + where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` + is the day number within the current year starting with ``1`` for January 1st. .. method:: date.toordinal() Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, + has ordinal 1. For any :class:`date` object *d*, ``date.fromordinal(d.toordinal()) == d``. @@ -592,26 +673,31 @@ Instance methods: Return a 3-tuple, (ISO year, ISO week number, ISO weekday). - The ISO calendar is a widely used variant of the Gregorian calendar. See - https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good - explanation. + The ISO calendar is a widely used variant of the Gregorian calendar. [#]_ The ISO year consists of 52 or 53 full weeks, and where a week starts on a - Monday and ends on a Sunday. The first week of an ISO year is the first + Monday and ends on a Sunday. The first week of an ISO year is the first (Gregorian) calendar week of a year containing a Thursday. This is called week number 1, and the ISO year of that Thursday is the same as its Gregorian year. For example, 2004 begins on a Thursday, so the first week of ISO year 2004 - begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so that - ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and ``date(2004, 1, - 4).isocalendar() == (2004, 1, 7)``. + begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: + >>> from datetime import date + >>> date(2003, 12, 29).isocalendar() + (2004, 1, 1) + >>> date(2004, 1, 4).isocalendar() + (2004, 1, 7) .. method:: date.isoformat() - Return a string representing the date in ISO 8601 format, 'YYYY-MM-DD'. For - example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``. + Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: + + >>> from datetime import date + >>> date(2002, 12, 4).isoformat() + '2002-12-04' + This is the inverse of :meth:`date.fromisoformat`. .. method:: date.__str__() @@ -620,9 +706,17 @@ Instance methods: .. method:: date.ctime() - Return a string representing the date, for example ``date(2002, 12, - 4).ctime() == 'Wed Dec 4 00:00:00 2002'``. ``d.ctime()`` is equivalent to - ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the native C + Return a string representing the date:: + + >>> from datetime import date + >>> date(2002, 12, 4).ctime() + 'Wed Dec 4 00:00:00 2002' + + ``d.ctime()`` is equivalent to:: + + time.ctime(time.mktime(d.timetuple())) + + on platforms where the native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which :meth:`date.ctime` does not invoke) conforms to the C standard. @@ -643,6 +737,8 @@ Instance methods: complete list of formatting directives, see :ref:`strftime-strptime-behavior`. +Examples of Usage: :class:`date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: @@ -662,7 +758,7 @@ Example of counting days to an event:: >>> time_to_birthday.days 202 -Example of working with :class:`date`: +More examples of working with :class:`date`: .. doctest:: @@ -670,6 +766,20 @@ Example of working with :class:`date`: >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) + + >>> # Methods related to formatting string output + >>> d.isoformat() + '2002-03-11' + >>> d.strftime("%d/%m/%y") + '11/03/02' + >>> d.strftime("%A %d. %B %Y") + 'Monday 11. March 2002' + >>> d.ctime() + 'Mon Mar 11 00:00:00 2002' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") + 'The day is 11, the month is March.' + + >>> # Methods for to extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -688,14 +798,10 @@ Example of working with :class:`date`: 2002 # ISO year 11 # ISO week number 1 # ISO day number ( 1 = Monday ) - >>> d.isoformat() - '2002-03-11' - >>> d.strftime("%d/%m/%y") - '11/03/02' - >>> d.strftime("%A %d. %B %Y") - 'Monday 11. March 2002' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") - 'The day is 11, the month is March.' + + >>> # A date object is immutable; all operations produce a new object + >>> d.replace(year=2005) + datetime.date(2005, 3, 11) .. _datetime-datetime: @@ -704,17 +810,18 @@ Example of working with :class:`date`: -------------------------- A :class:`.datetime` object is a single object containing all the information -from a :class:`date` object and a :class:`.time` object. Like a :class:`date` -object, :class:`.datetime` assumes the current Gregorian calendar extended in -both directions; like a time object, :class:`.datetime` assumes there are exactly -3600\*24 seconds in every day. +from a :class:`date` object and a :class:`.time` object. + +Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian +calendar extended in both directions; like a :class:`.time` object, +:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: .. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - The year, month and day arguments are required. *tzinfo* may be ``None``, or an - instance of a :class:`tzinfo` subclass. The remaining arguments may be integers, + The *year*, *month* and *day* arguments are required. *tzinfo* may be ``None``, or an + instance of a :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``MINYEAR <= year <= MAXYEAR``, @@ -735,31 +842,48 @@ Other constructors, all class methods: .. classmethod:: datetime.today() - Return the current local datetime, with :attr:`.tzinfo` ``None``. This is - equivalent to ``datetime.fromtimestamp(time.time())``. See also :meth:`now`, - :meth:`fromtimestamp`. + Return the current local datetime, with :attr:`.tzinfo` ``None``. + + Equivalent to:: + + datetime.fromtimestamp(time.time()) + See also :meth:`now`, :meth:`fromtimestamp`. + + This method is functionally equivalent to :meth:`now`, but without a + ``tz`` parameter. .. classmethod:: datetime.now(tz=None) - Return the current local date and time. If optional argument *tz* is ``None`` + Return the current local date and time. + + If optional argument *tz* is ``None`` or not specified, this is like :meth:`today`, but, if possible, supplies more precision than can be gotten from going through a :func:`time.time` timestamp (for example, this may be possible on platforms supplying the C :c:func:`gettimeofday` function). - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - current date and time are converted to *tz*’s time zone. In this case the - result is equivalent to ``tz.fromutc(datetime.utcnow().replace(tzinfo=tz))``. - See also :meth:`today`, :meth:`utcnow`. + If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, + and the current date and time are converted to *tz*’s time zone. + + This function is preferred over :meth:`today` and :meth:`utcnow`. .. classmethod:: datetime.utcnow() - Return the current UTC date and time, with :attr:`.tzinfo` ``None``. This is like - :meth:`now`, but returns the current UTC date and time, as a naive - :class:`.datetime` object. An aware current UTC datetime can be obtained by - calling ``datetime.now(timezone.utc)``. See also :meth:`now`. + Return the current UTC date and time, with :attr:`.tzinfo` ``None``. + + This is like :meth:`now`, but returns the current UTC date and time, as a naive + :class:`.datetime` object. An aware current UTC datetime can be obtained by + calling ``datetime.now(timezone.utc)``. See also :meth:`now`. + + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC. As such, the recommended way to create an object representing the + current time in UTC is by calling ``datetime.now(timezone.utc)``. + .. classmethod:: datetime.fromtimestamp(timestamp, tz=None) @@ -769,9 +893,7 @@ Other constructors, all class methods: the returned :class:`.datetime` object is naive. If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - timestamp is converted to *tz*’s time zone. In this case the result is - equivalent to - ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``. + timestamp is converted to *tz*’s time zone. :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of the range of values supported by the platform C :c:func:`localtime` or @@ -781,7 +903,8 @@ Other constructors, all class methods: 1970 through 2038. Note that on non-POSIX systems that include leap seconds in their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`, and then it's possible to have two timestamps differing by a second that yield - identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`. + identical :class:`.datetime` objects. This method is preferred over + :meth:`utcfromtimestamp`. .. versionchanged:: 3.3 Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp @@ -796,7 +919,9 @@ Other constructors, all class methods: .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with - :attr:`.tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is + :attr:`.tzinfo` ``None``. (The resulting object is naive.) + + This may raise :exc:`OverflowError`, if the timestamp is out of the range of values supported by the platform C :c:func:`gmtime` function, and :exc:`OSError` on :c:func:`gmtime` failure. It's common for this to be restricted to years in 1970 through 2038. @@ -813,6 +938,14 @@ Other constructors, all class methods: except the latter formula always supports the full years range: between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC. As such, the recommended way to create an object representing a + specific timestamp in UTC is by calling + ``datetime.fromtimestamp(timestamp, tz=timezone.utc)``. + .. versionchanged:: 3.3 Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp is out of the range of values supported by the platform C @@ -824,7 +957,7 @@ Other constructors, all class methods: Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and + <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. @@ -832,13 +965,13 @@ Other constructors, all class methods: Return a new :class:`.datetime` object whose date components are equal to the given :class:`date` object's, and whose time components - are equal to the given :class:`.time` object's. If the *tzinfo* + are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument is used. For any :class:`.datetime` object *d*, - ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a + ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. @@ -848,23 +981,44 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisoformat(date_string) - Return a :class:`datetime` corresponding to a *date_string* in one of the - formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. - Specifically, this function supports strings in the format(s) - ``YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]]``, - where ``*`` can match any single character. + Return a :class:`.datetime` corresponding to a *date_string* in one of the + formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. + + Specifically, this function supports strings in the format: + + .. code-block:: none + + YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]] - .. caution:: + where ``*`` can match any single character. - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`datetime.isoformat`. + .. caution:: - .. versionadded:: 3.7 + This does *not* support parsing arbitrary ISO 8601 strings - it is only intended + as the inverse operation of :meth:`datetime.isoformat`. A more full-featured + ISO 8601 parser, ``dateutil.parser.isoparse`` is available in the third-party package + `dateutil `__. + Examples:: + + >>> from datetime import datetime + >>> datetime.fromisoformat('2011-11-04') + datetime.datetime(2011, 11, 4, 0, 0) + >>> datetime.fromisoformat('2011-11-04T00:05:23') + datetime.datetime(2011, 11, 4, 0, 5, 23) + >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) + >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) + >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + datetime.datetime(2011, 11, 4, 0, 5, 23, + tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) + + .. versionadded:: 3.7 .. classmethod:: datetime.fromisocalendar(year, week, day) - Return a :class:`datetime` corresponding to the ISO calendar date specified + Return a :class:`.datetime` corresponding to the ISO calendar date specified by year, week and day. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. @@ -874,8 +1028,13 @@ Other constructors, all class methods: .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to - *format*. This is equivalent to ``datetime(*(time.strptime(date_string, - format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format + *format*. + + This is equivalent to:: + + datetime(*(time.strptime(date_string, format)[0:6])) + + :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a time tuple. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -947,7 +1106,7 @@ Instance attributes (read-only): .. attribute:: datetime.fold - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A + In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) The value 0 (1) represents the earlier (later) of the two moments with the same wall @@ -972,7 +1131,7 @@ Supported operations: (1) datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The + time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if datetime2.year would be smaller than :const:`MINYEAR` or larger than @@ -986,16 +1145,16 @@ Supported operations: (3) Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if - both operands are naive, or if both are aware. If one is aware and the other is + both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments + object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments are done in this case. If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The + as if *a* and *b* were first converted to naive UTC datetimes first. The result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - b.utcoffset())`` except that the implementation never overflows. @@ -1004,34 +1163,31 @@ Supported operations: *datetime2* in time. If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality + is raised if an order comparison is attempted. For equality comparisons, naive instances are never equal to aware instances. If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` + compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparands are first adjusted by subtracting their UTC offsets (obtained from ``self.utcoffset()``). .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`.datetime` + Equality comparisons between aware and naive :class:`.datetime` instances don't raise :exc:`TypeError`. .. note:: In order to stop comparison from falling back to the default scheme of comparing object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, + other comparand isn't also a :class:`.datetime` object. However, ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` + :meth:`timetuple` attribute. This hook gives other kinds of date objects a + chance at implementing mixed-type comparison. If not, when a :class:`.datetime` object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return + unless the comparison is ``==`` or ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -:class:`.datetime` objects can be used as dictionary keys. In Boolean contexts, -all :class:`.datetime` objects are considered to be true. - Instance methods: .. method:: datetime.date() @@ -1042,7 +1198,7 @@ Instance methods: .. method:: datetime.time() Return :class:`.time` object with same hour, minute, second, microsecond and fold. - :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. + :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. .. versionchanged:: 3.6 The fold value is copied to the returned :class:`.time` object. @@ -1051,7 +1207,7 @@ Instance methods: .. method:: datetime.timetz() Return :class:`.time` object with same hour, minute, second, microsecond, fold, and - tzinfo attributes. See also method :meth:`time`. + tzinfo attributes. See also method :meth:`time`. .. versionchanged:: 3.6 The fold value is copied to the returned :class:`.time` object. @@ -1059,10 +1215,10 @@ Instance methods: .. method:: datetime.replace(year=self.year, month=self.month, day=self.day, \ hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ - tzinfo=self.tzinfo, * fold=0) + tzinfo=self.tzinfo, *, fold=0) Return a datetime with the same attributes, except for those attributes given - new values by whichever keyword arguments are specified. Note that + new values by whichever keyword arguments are specified. Note that ``tzinfo=None`` can be specified to create a naive datetime from an aware datetime with no conversion of date and time data. @@ -1077,11 +1233,11 @@ Instance methods: *self*, but in *tz*'s local time. If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its - :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* + :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* is naive, it is presumed to represent time in the system timezone. If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted + timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted datetime instance will be set to an instance of :class:`timezone` with the zone name and offset obtained from the OS. @@ -1092,7 +1248,7 @@ Instance methods: the same date and time data as ``dt - dt.utcoffset()``. If you merely want to attach a time zone object *tz* to a datetime *dt* without - adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you + adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you merely want to remove the time zone object from an aware datetime *dt* without conversion of date and time data, use ``dt.replace(tzinfo=None)``. @@ -1146,44 +1302,58 @@ Instance methods: .. method:: datetime.timetuple() Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - ``d.timetuple()`` is equivalent to ``time.struct_time((d.year, d.month, d.day, - d.hour, d.minute, d.second, d.weekday(), yday, dst))``, where ``yday = - d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within - the current year starting with ``1`` for January 1st. The :attr:`tm_isdst` flag - of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is - ``None`` or :meth:`dst` returns ``None``, :attr:`tm_isdst` is set to ``-1``; - else if :meth:`dst` returns a non-zero value, :attr:`tm_isdst` is set to ``1``; - else :attr:`tm_isdst` is set to ``0``. + + ``d.timetuple()`` is equivalent to:: + + time.struct_time((d.year, d.month, d.day, + d.hour, d.minute, d.second, + d.weekday(), yday, dst)) + + where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` + is the day number within the current year starting with ``1`` for January + 1st. The :attr:`tm_isdst` flag of the result is set according to the + :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns + ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a + non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is + set to ``0``. .. method:: datetime.utctimetuple() If :class:`.datetime` instance *d* is naive, this is the same as ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what - ``d.dst()`` returns. DST is never in effect for a UTC time. + ``d.dst()`` returns. DST is never in effect for a UTC time. If *d* is aware, *d* is normalized to UTC time, by subtracting ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note + normalized time is returned. :attr:`tm_isdst` is forced to 0. Note that an :exc:`OverflowError` may be raised if *d*.year was ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year boundary. + .. warning:: + + Because naive ``datetime`` objects are treated by many ``datetime`` methods + as local times, it is preferred to use aware datetimes to represent times + in UTC; as a result, using ``utcfromtimetuple`` may give misleading + results. If you have a naive ``datetime`` representing UTC, use + ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point + you can use :meth:`.datetime.timetuple`. .. method:: datetime.toordinal() - Return the proleptic Gregorian ordinal of the date. The same as + Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` - instance. The return value is a :class:`float` similar to that + instance. The return value is a :class:`float` similar to that returned by :func:`time.time`. Naive :class:`.datetime` instances are assumed to represent local time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` + function to perform the conversion. Since :class:`.datetime` supports wider range of values than :c:func:`mktime` on many platforms, this method may raise :exc:`OverflowError` for times far in the past or far in the future. @@ -1202,7 +1372,7 @@ Instance methods: .. note:: There is no method to obtain the POSIX timestamp directly from a - naive :class:`.datetime` instance representing UTC time. If your + naive :class:`.datetime` instance representing UTC time. If your application uses this convention and your system timezone is not set to UTC, you can obtain the POSIX timestamp by supplying ``tzinfo=timezone.utc``:: @@ -1228,30 +1398,45 @@ Instance methods: .. method:: datetime.isocalendar() - Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as + Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as ``self.date().isocalendar()``. .. method:: datetime.isoformat(sep='T', timespec='auto') - Return a string representing the date and time in ISO 8601 format, - YYYY-MM-DDTHH:MM:SS.ffffff or, if :attr:`microsecond` is 0, - YYYY-MM-DDTHH:MM:SS + Return a string representing the date and time in ISO 8601 format: + + - ``YYYY-MM-DDTHH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 + - ``YYYY-MM-DDTHH:MM:SS``, if :attr:`microsecond` is 0 If :meth:`utcoffset` does not return ``None``, a string is appended, giving the UTC offset: - YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] or, if :attr:`microsecond` - is 0 YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]. + + - ``YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` + is not 0 + - ``YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 + + Examples:: + + >>> from datetime import datetime, timezone + >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + '2019-05-18T15:17:08.132263' + >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, - placed between the date and time portions of the result. For example, + placed between the date and time portions of the result. For example:: >>> from datetime import tzinfo, timedelta, datetime >>> class TZ(tzinfo): - ... def utcoffset(self, dt): return timedelta(minutes=-399) + ... """A time zone with an arbitrary, constant -06:39 offset.""" + ... def utcoffset(self, dt): + ... return timedelta(hours=-6, minutes=-39) ... >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' + >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional components of the time to include (the default is ``'auto'``). @@ -1259,19 +1444,19 @@ Instance methods: - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. + in ``HH:MM:SS`` format. - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. + part to milliseconds. ``HH:MM:SS.sss`` format. + - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. .. note:: Excluded time components are truncated, not rounded. - :exc:`ValueError` will be raised on an invalid *timespec* argument. + :exc:`ValueError` will be raised on an invalid *timespec* argument:: >>> from datetime import datetime @@ -1293,48 +1478,64 @@ Instance methods: .. method:: datetime.ctime() - Return a string representing the date and time, for example ``datetime(2002, 12, - 4, 20, 30, 40).ctime() == 'Wed Dec 4 20:30:40 2002'``. ``d.ctime()`` is - equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the - native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which - :meth:`datetime.ctime` does not invoke) conforms to the C standard. + Return a string representing the date and time:: + + >>> from datetime import datetime + >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + 'Wed Dec 4 20:30:40 2002' + + The output string will *not* include time zone information, regardless + of whether the input is aware or naive. + ``d.ctime()`` is equivalent to:: + + time.ctime(time.mktime(d.timetuple())) + + on platforms where the native C :c:func:`ctime` function + (which :func:`time.ctime` invokes, but which + :meth:`datetime.ctime` does not invoke) conforms to the C standard. .. method:: datetime.strftime(format) Return a string representing the date and time, controlled by an explicit format - string. For a complete list of formatting directives, see + string. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. .. method:: datetime.__format__(format) - Same as :meth:`.datetime.strftime`. This makes it possible to specify a format + Same as :meth:`.datetime.strftime`. This makes it possible to specify a format string for a :class:`.datetime` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. +Examples of Usage: :class:`.datetime` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Examples of working with datetime objects: +Examples of working with :class:`~datetime.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time + >>> from datetime import datetime, date, time, timezone + >>> # Using datetime.combine() >>> d = date(2005, 7, 14) >>> t = time(12, 30) >>> datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) - >>> # Using datetime.now() or datetime.utcnow() + + >>> # Using datetime.now() >>> datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.utcnow() # doctest: +SKIP - datetime.datetime(2007, 12, 6, 15, 29, 43, 79060) + >>> datetime.now(timezone.utc) # doctest: +SKIP + datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) + >>> # Using datetime.strptime() >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") >>> dt datetime.datetime(2006, 11, 21, 16, 30) + >>> # Using datetime.timetuple() to get tuple of all attributes >>> tt = dt.timetuple() >>> for it in tt: # doctest: +SKIP @@ -1349,6 +1550,7 @@ Examples of working with datetime objects: 1 # weekday (0 = Monday) 325 # number of days since 1st January -1 # dst - method tzinfo.dst() returned None + >>> # Date in ISO format >>> ic = dt.isocalendar() >>> for it in ic: # doctest: +SKIP @@ -1357,89 +1559,95 @@ Examples of working with datetime objects: 2006 # ISO year 47 # ISO week 2 # ISO weekday - >>> # Formatting datetime + + >>> # Formatting a datetime >>> dt.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' -Using datetime with tzinfo: - - >>> from datetime import timedelta, datetime, tzinfo, timezone - >>> class KabulTz(tzinfo): - ... # Kabul used +4 until 1945, when they moved to +4:30 - ... UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) - ... def utcoffset(self, dt): - ... if dt.year < 1945: - ... return timedelta(hours=4) - ... elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): - ... # If dt falls in the imaginary range, use fold to decide how - ... # to resolve. See PEP495 - ... return timedelta(hours=4, minutes=(30 if dt.fold else 0)) - ... else: - ... return timedelta(hours=4, minutes=30) - ... - ... def fromutc(self, dt): - ... # A custom implementation is required for fromutc as - ... # the input to this function is a datetime with utc values - ... # but with a tzinfo set to self - ... # See datetime.astimezone or fromtimestamp - ... - ... # Follow same validations as in datetime.tzinfo - ... if not isinstance(dt, datetime): - ... raise TypeError("fromutc() requires a datetime argument") - ... if dt.tzinfo is not self: - ... raise ValueError("dt.tzinfo is not self") - ... - ... if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - ... return dt + timedelta(hours=4, minutes=30) - ... else: - ... return dt + timedelta(hours=4) - ... - ... def dst(self, dt): - ... return timedelta(0) - ... - ... def tzname(self, dt): - ... if dt >= self.UTC_MOVE_DATE: - ... return "+04:30" - ... else: - ... return "+04" - ... - ... def __repr__(self): - ... return f"{self.__class__.__name__}()" - ... - >>> tz1 = KabulTz() - >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) - >>> print(dt1.utcoffset()) - 4:00:00 - >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) - >>> print(dt2.utcoffset()) - 4:30:00 - >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) - >>> dt3 - datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) - >>> dt2 - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) - >>> dt2.utctimetuple() == dt3.utctimetuple() - True - - +The example below defines a :class:`tzinfo` subclass capturing time zone +information for Kabul, Afghanistan, which used +4 UTC until 1945 +and then +4:30 UTC thereafter:: + + from datetime import timedelta, datetime, tzinfo, timezone + + class KabulTz(tzinfo): + # Kabul used +4 until 1945, when they moved to +4:30 + UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + + def utcoffset(self, dt): + if dt.year < 1945: + return timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + # An ambiguous ("imaginary") half-hour range representing + # a 'fold' in time due to the shift from +4 to +4:30. + # If dt falls in the imaginary range, use fold to decide how + # to resolve. See PEP495. + return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + else: + return timedelta(hours=4, minutes=30) + + def fromutc(self, dt): + # Follow same validations as in datetime.tzinfo + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + # A custom implementation is required for fromutc as + # the input to this function is a datetime with utc values + # but with a tzinfo set to self. + # See datetime.astimezone or fromtimestamp. + if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: + return dt + timedelta(hours=4, minutes=30) + else: + return dt + timedelta(hours=4) + + def dst(self, dt): + # Kabul does not observe daylight saving time. + return timedelta(0) + + def tzname(self, dt): + if dt >= self.UTC_MOVE_DATE: + return "+04:30" + return "+04" + +Usage of ``KabulTz`` from above:: + + >>> tz1 = KabulTz() + + >>> # Datetime before the change + >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> print(dt1.utcoffset()) + 4:00:00 + + >>> # Datetime after the change + >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> print(dt2.utcoffset()) + 4:30:00 + + >>> # Convert datetime to another time zone + >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 + datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) + >>> dt2 + datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) + >>> dt2 == dt3 + True .. _datetime-time: :class:`.time` Objects ---------------------- -A time object represents a (local) time of day, independent of any particular +A :class:`time` object represents a (local) time of day, independent of any particular day, and subject to adjustment via a :class:`tzinfo` object. .. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - All arguments are optional. *tzinfo* may be ``None``, or an instance of a - :class:`tzinfo` subclass. The remaining arguments may be integers, in the + All arguments are optional. *tzinfo* may be ``None``, or an instance of a + :class:`tzinfo` subclass. The remaining arguments must be integers in the following ranges: * ``0 <= hour < 24``, @@ -1448,7 +1656,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``0 <= microsecond < 1000000``, * ``fold in [0, 1]``. - If an argument outside those ranges is given, :exc:`ValueError` is raised. All + If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to ``0`` except *tzinfo*, which defaults to :const:`None`. Class attributes: @@ -1501,7 +1709,7 @@ Instance attributes (read-only): .. attribute:: time.fold - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A + In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A repeated interval occurs when clocks are rolled back at the end of daylight saving time or when the UTC offset for the current zone is decreased for political reasons.) The value 0 (1) represents the earlier (later) of the two moments with the same wall @@ -1509,38 +1717,32 @@ Instance attributes (read-only): .. versionadded:: 3.6 +:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, +where *a* is considered less +than *b* when *a* precedes *b* in time. If one comparand is naive and the other +is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality +comparisons, naive instances are never equal to aware instances. -Supported operations: - -* comparison of :class:`.time` to :class:`.time`, where *a* is considered less - than *b* when *a* precedes *b* in time. If one comparand is naive and the other - is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have - the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is - ignored and the base times are compared. If both comparands are aware and - have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by - subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order - to stop mixed-type comparisons from falling back to the default comparison by - object address, when a :class:`.time` object is compared to an object of a - different type, :exc:`TypeError` is raised unless the comparison is ``==`` or - ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. - - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`~datetime.time` instances - don't raise :exc:`TypeError`. +If both comparands are aware, and have +the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is +ignored and the base times are compared. If both comparands are aware and +have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by +subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order +to stop mixed-type comparisons from falling back to the default comparison by +object address, when a :class:`.time` object is compared to an object of a +different type, :exc:`TypeError` is raised unless the comparison is ``==`` or +``!=``. The latter cases return :const:`False` or :const:`True`, respectively. -* hash, use as dict key +.. versionchanged:: 3.3 + Equality comparisons between aware and naive :class:`~datetime.time` instances + don't raise :exc:`TypeError`. -* efficient pickling - -In boolean contexts, a :class:`.time` object is always considered to be true. +In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it - represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full + represented midnight in UTC. This behavior was considered obscure and + error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. @@ -1548,25 +1750,39 @@ Other constructor: .. classmethod:: time.fromisoformat(time_string) - Return a :class:`time` corresponding to a *time_string* in one of the - formats emitted by :meth:`time.isoformat`. Specifically, this function supports - strings in the format(s) ``HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]``. + Return a :class:`.time` corresponding to a *time_string* in one of the + formats emitted by :meth:`time.isoformat`. Specifically, this function supports + strings in the format: + + .. code-block:: none + + HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] + + .. caution:: + + This does *not* support parsing arbitrary ISO 8601 strings. It is only + intended as the inverse operation of :meth:`time.isoformat`. - .. caution:: + Examples:: - This does not support parsing arbitrary ISO 8601 strings - it is only intended - as the inverse operation of :meth:`time.isoformat`. + >>> from datetime import time + >>> time.fromisoformat('04:23:01') + datetime.time(4, 23, 1) + >>> time.fromisoformat('04:23:01.000384') + datetime.time(4, 23, 1, 384) + >>> time.fromisoformat('04:23:01+04:00') + datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - .. versionadded:: 3.7 + .. versionadded:: 3.7 Instance methods: .. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ - microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0) + microsecond=self.microsecond, tzinfo=self.tzinfo, *, fold=0) Return a :class:`.time` with the same value, except for those attributes given - new values by whichever keyword arguments are specified. Note that + new values by whichever keyword arguments are specified. Note that ``tzinfo=None`` can be specified to create a naive :class:`.time` from an aware :class:`.time`, without conversion of the time data. @@ -1576,10 +1792,12 @@ Instance methods: .. method:: time.isoformat(timespec='auto') - Return a string representing the time in ISO 8601 format, HH:MM:SS.ffffff or, if - :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a - string is appended, giving the UTC offset: HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]] - or, if self.microsecond is 0, HH:MM:SS+HH:MM[:SS[.ffffff]]. + Return a string representing the time in ISO 8601 format, one of: + + - ``HH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 + - ``HH:MM:SS``, if :attr:`microsecond` is 0 + - ``HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :meth:`utcoffset` does not return ``None`` + - ``HH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 and :meth:`utcoffset` does not return ``None`` The optional argument *timespec* specifies the number of additional components of the time to include (the default is ``'auto'``). @@ -1587,13 +1805,13 @@ Instance methods: - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. + in ``HH:MM:SS`` format. - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.ffffff format. + part to milliseconds. ``HH:MM:SS.sss`` format. + - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. .. note:: @@ -1601,6 +1819,7 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument. + Example:: >>> from datetime import time >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') @@ -1623,7 +1842,7 @@ Instance methods: .. method:: time.strftime(format) Return a string representing the time, controlled by an explicit format - string. For a complete list of formatting directives, see + string. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1631,7 +1850,7 @@ Instance methods: Same as :meth:`.time.strftime`. This makes it possible to specify a format string for a :class:`.time` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1661,7 +1880,10 @@ Instance methods: ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Example: +Examples of Usage: :class:`.time` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of working with a :class:`.time` object:: >>> from datetime import time, tzinfo, timedelta >>> class TZ1(tzinfo): @@ -1697,12 +1919,8 @@ Example: .. class:: tzinfo() This is an abstract base class, meaning that this class should not be - instantiated directly. You need to derive a concrete subclass, and (at least) - supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module supplies - a simple concrete subclass of :class:`tzinfo`, :class:`timezone`, which can represent - timezones with fixed offset from UTC such as UTC itself or North American EST and - EDT. + instantiated directly. Define a subclass of :class:`tzinfo` to capture + information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects @@ -1710,28 +1928,35 @@ Example: supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. + You need to derive a concrete subclass, and (at least) + supply implementations of the standard :class:`tzinfo` methods needed by the + :class:`.datetime` methods you use. The :mod:`datetime` module provides + :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + represent timezones with fixed offset from UTC such as UTC itself or North + American EST and EDT. + Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, else it can be - pickled but possibly not unpickled again. This is a technical requirement that + :meth:`__init__` method that can be called with no arguments, otherwise it can be + pickled but possibly not unpickled again. This is a technical requirement that may be relaxed in the future. A concrete subclass of :class:`tzinfo` may need to implement the following - methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. + methods. Exactly which methods are needed depends on the uses made of aware + :mod:`datetime` objects. If in doubt, simply implement all of them. .. method:: tzinfo.utcoffset(dt) Return offset of local time from UTC, as a :class:`timedelta` object that is - positive east of UTC. If local time is - west of UTC, this should be negative. Note that this is intended to be the - total offset from UTC; for example, if a :class:`tzinfo` object represents both - time zone and DST adjustments, :meth:`utcoffset` should return their sum. If - the UTC offset isn't known, return ``None``. Else the value returned must be a - :class:`timedelta` object strictly between ``-timedelta(hours=24)`` and - ``timedelta(hours=24)`` (the magnitude of the offset must be less - than one day). Most implementations of :meth:`utcoffset` will probably look - like one of these two:: + positive east of UTC. If local time is west of UTC, this should be negative. + + This represents the *total* offset from UTC; for example, if a + :class:`tzinfo` object represents both time zone and DST adjustments, + :meth:`utcoffset` should return their sum. If the UTC offset isn't known, + return ``None``. Else the value returned must be a :class:`timedelta` object + strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)`` + (the magnitude of the offset must be less than one day). Most implementations + of :meth:`utcoffset` will probably look like one of these two:: return CONSTANT # fixed-offset class return CONSTANT + self.dst(dt) # daylight-aware class @@ -1750,12 +1975,14 @@ Example: Return the daylight saving time (DST) adjustment, as a :class:`timedelta` object or - ``None`` if DST information isn't known. Return ``timedelta(0)`` if DST is not - in effect. If DST is in effect, return the offset as a :class:`timedelta` object + ``None`` if DST information isn't known. + + Return ``timedelta(0)`` if DST is not in effect. + If DST is in effect, return the offset as a :class:`timedelta` object (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has already been added to the UTC offset returned by :meth:`utcoffset`, so there's no need to consult :meth:`dst` unless you're interested in obtaining DST info - separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` + separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for DST changes when crossing time zones. @@ -1768,9 +1995,9 @@ Example: must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == tz`` For sane :class:`tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but - only on geographic location. The implementation of :meth:`datetime.astimezone` + only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. @@ -1780,12 +2007,12 @@ Example: # a fixed-offset class: doesn't account for DST return timedelta(0) - or :: + or:: def dst(self, dt): # Code to set dston and dstoff to the time zone's DST # transition times based on the input dt.year, and expressed - # in standard local time. Then + # in standard local time. if dston <= dt.replace(tzinfo=None) < dstoff: return timedelta(hours=1) @@ -1802,9 +2029,9 @@ Example: Return the time zone name corresponding to the :class:`.datetime` object *dt*, as a string. Nothing about string names is defined by the :mod:`datetime` module, - and there's no requirement that it mean anything in particular. For example, + and there's no requirement that it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all - valid replies. Return ``None`` if a string name isn't known. Note that this is + valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value of *dt* passed, especially if the :class:`tzinfo` class is accounting for @@ -1814,20 +2041,20 @@ Example: These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes +response to their methods of the same names. A :class:`.datetime` object passes itself as the argument, and a :class:`.time` object passes ``None`` as the -argument. A :class:`tzinfo` subclass's methods should therefore be prepared to +argument. A :class:`tzinfo` subclass's methods should therefore be prepared to accept a *dt* argument of ``None``, or of class :class:`.datetime`. When ``None`` is passed, it's up to the class designer to decide the best -response. For example, returning ``None`` is appropriate if the class wishes to -say that time objects don't participate in the :class:`tzinfo` protocols. It +response. For example, returning ``None`` is appropriate if the class wishes to +say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. When a :class:`.datetime` object is passed in response to a :class:`.datetime` -method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The +method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can +rely on this, unless user code calls :class:`tzinfo` methods directly. The intent is that the :class:`tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other timezones. @@ -1837,16 +2064,16 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: .. method:: tzinfo.fromutc(dt) This is called from the default :class:`datetime.astimezone()` - implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s - date and time data are to be viewed as expressing a UTC time. The purpose + implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s + date and time data are to be viewed as expressing a UTC time. The purpose of :meth:`fromutc` is to adjust the date and time data, returning an equivalent datetime in *self*'s local time. Most :class:`tzinfo` subclasses should be able to inherit the default - :meth:`fromutc` implementation without problems. It's strong enough to handle + :meth:`fromutc` implementation without problems. It's strong enough to handle fixed-offset time zones, and time zones accounting for both standard and daylight time, and the latter even if the DST transition times differ in - different years. An example of a time zone the default :meth:`fromutc` + different years. An example of a time zone the default :meth:`fromutc` implementation may not handle correctly in all cases is one where the standard offset (from UTC) depends on the specific date and time passed, which can happen for political reasons. The default implementations of :meth:`astimezone` and @@ -1879,7 +2106,7 @@ In the following :download:`tzinfo_examples.py Note that there are unavoidable subtleties twice per year in a :class:`tzinfo` subclass accounting for both standard and daylight time, at the DST transition -points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the +points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the minute after 1:59 (EST) on the second Sunday in March, and ends the minute after 1:59 (EDT) on the first Sunday in November:: @@ -1892,9 +2119,9 @@ minute after 1:59 (EST) on the second Sunday in March, and ends the minute after end 23:MM 0:MM 1:MM 1:MM 2:MM 3:MM When DST starts (the "start" line), the local wall clock leaps from 1:59 to -3:00. A wall time of the form 2:MM doesn't really make sense on that day, so +3:00. A wall time of the form 2:MM doesn't really make sense on that day, so ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST -begins. For example, at the Spring forward transition of 2016, we get +begins. For example, at the Spring forward transition of 2016, we get:: >>> from datetime import datetime, timezone >>> from tzinfo_examples import HOUR, Eastern @@ -1912,14 +2139,14 @@ begins. For example, at the Spring forward transition of 2016, we get When DST ends (the "end" line), there's a potentially worse problem: there's an hour that can't be spelled unambiguously in local wall time: the last hour of -daylight time. In Eastern, that's times of the form 5:MM UTC on the day -daylight time ends. The local wall clock leaps from 1:59 (daylight time) back +daylight time. In Eastern, that's times of the form 5:MM UTC on the day +daylight time ends. The local wall clock leaps from 1:59 (daylight time) back to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. :meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC -hours into the same local hour then. In the Eastern example, UTC times of the +hours into the same local hour then. In the Eastern example, UTC times of the form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. -For example, at the Fall back transition of 2016, we get +For example, at the Fall back transition of 2016, we get:: >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) >>> for i in range(4): @@ -1932,7 +2159,7 @@ For example, at the Fall back transition of 2016, we get 06:00:00 UTC = 01:00:00 EST 1 07:00:00 UTC = 02:00:00 EST 0 -Note that the :class:`datetime` instances that differ only by the value of the +Note that the :class:`.datetime` instances that differ only by the value of the :attr:`~datetime.fold` attribute are considered equal in comparisons. Applications that can't bear wall-time ambiguities should explicitly check the @@ -1944,15 +2171,17 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: `dateutil.tz `_ - The standard library has :class:`timezone` class for handling arbitrary - fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. + The :mod:`datetime` module has a basic :class:`timezone` class (for + handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` + attribute (a UTC timezone instance). - *dateutil.tz* library brings the *IANA timezone database* (also known as the - Olson database) to Python and its usage is recommended. + *dateutil.tz* library brings the *IANA timezone database* + (also known as the Olson database) to Python, and its usage is + recommended. `IANA timezone database `_ - The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and - data that represent the history of local time for many representative + The Time Zone Database (often called tz, tzdata or zoneinfo) contains code + and data that represent the history of local time for many representative locations around the globe. It is updated periodically to reflect changes made by political bodies to time zone boundaries, UTC offsets, and daylight-saving rules. @@ -1965,20 +2194,21 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a timezone defined by a fixed offset from -UTC. Note that objects of this class cannot be used to represent -timezone information in the locations where different offsets are used -in different days of the year or where historical changes have been -made to civil time. +UTC. + +Objects of this class cannot be used to represent timezone information in the +locations where different offsets are used in different days of the year or +where historical changes have been made to civil time. .. class:: timezone(offset, name=None) The *offset* argument must be specified as a :class:`timedelta` - object representing the difference between the local time and UTC. It must + object representing the difference between the local time and UTC. It must be strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)``, otherwise :exc:`ValueError` is raised. - The *name* argument is optional. If specified it must be a string that + The *name* argument is optional. If specified it must be a string that will be used as the value returned by the :meth:`datetime.tzname` method. .. versionadded:: 3.2 @@ -1990,9 +2220,10 @@ made to civil time. .. method:: timezone.utcoffset(dt) Return the fixed value specified when the :class:`timezone` instance is - constructed. The *dt* argument is ignored. The return value is a - :class:`timedelta` instance equal to the difference between the - local time and UTC. + constructed. + + The *dt* argument is ignored. The return value is a :class:`timedelta` + instance equal to the difference between the local time and UTC. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. @@ -2000,16 +2231,17 @@ made to civil time. .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance - is constructed. If *name* is not provided in the constructor, the - name returned by ``tzname(dt)`` is generated from the value of the - ``offset`` as follows. If *offset* is ``timedelta(0)``, the name - is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign - of ``offset``, HH and MM are two digits of ``offset.hours`` and - ``offset.minutes`` respectively. + is constructed. + + If *name* is not provided in the constructor, the name returned by + ``tzname(dt)`` is generated from the value of the ``offset`` as follows. If + *offset* is ``timedelta(0)``, the name is "UTC", otherwise it is a string in + the format ``UTC±HH:MM``, where ± is the sign of ``offset``, HH and MM are + two digits of ``offset.hours`` and ``offset.minutes`` respectively. .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. + Name generated from ``offset=timedelta(0)`` is now plain `'UTC'`, not + ``'UTC+00:00'``. .. method:: timezone.dst(dt) @@ -2018,7 +2250,7 @@ made to civil time. .. method:: timezone.fromutc(dt) - Return ``dt + offset``. The *dt* argument must be an aware + Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. Class attributes: @@ -2038,43 +2270,33 @@ Class attributes: :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the -control of an explicit format string. Broadly speaking, ``d.strftime(fmt)`` -acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` -although not all objects support a :meth:`timetuple` method. +control of an explicit format string. Conversely, the :meth:`datetime.strptime` class method creates a :class:`.datetime` object from a string representing a date and time and a -corresponding format string. ``datetime.strptime(date_string, format)`` is -equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``, except -when the format includes sub-second components or timezone offset information, -which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. +corresponding format string. -For :class:`.time` objects, the format codes for year, month, and day should not -be used, as time objects have no such values. If they're used anyway, ``1900`` -is substituted for the year, and ``1`` for the month and day. - -For :class:`date` objects, the format codes for hours, minutes, seconds, and -microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. +The table below provides a high-level comparison of :meth:`strftime` +versus :meth:`strptime`: -For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``: -any components not specified in the format string will be pulled from the default value. [#]_ ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| | ``strftime`` | ``strptime`` | ++================+========================================================+==============================================================================+ +| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Type of method | Instance method | Class method | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ +| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | ++----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform -variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. -For the same reason, handling of format strings containing Unicode code points -that can't be represented in the charset of the current locale is also -platform-dependent. On some platforms such code points are preserved intact in -the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return -an empty string instead. +:meth:`strftime` and :meth:`strptime` Format Codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The following is a list of all the format codes that the C standard (1989 -version) requires, and these work on all platforms with a standard C -implementation. Note that the 1999 version of the C standard added additional -format codes. +The following is a list of all the format codes that the 1989 C standard +requires, and these work on all platforms with a standard C implementation. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | @@ -2093,7 +2315,7 @@ format codes. | | where 0 is Sunday and 6 is | | | | | Saturday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | @@ -2106,55 +2328,55 @@ format codes. | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | | | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | | | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | | | number, zero-padded on the | 999999 | | | | left. | | | +-----------+--------------------------------+------------------------+-------+ | ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ±HHMM[SS[.ffffff]] (empty | -0400, +1030, | | +| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | | | string if the object is | +063415, | | | | naive). | -030712.345216 | | +-----------+--------------------------------+------------------------+-------+ | ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | | | if the object is naive). | | | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Sunday as the first day of | | \(9) | | | the week) as a zero padded | | | | | decimal number. All days in a | | | | | new year preceding the first | | | | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| | (Monday as the first day of | | \(9) | | | the week) as a decimal number. | | | | | All days in a new year | | | | | preceding the first Monday | | | @@ -2177,11 +2399,7 @@ format codes. +-----------+--------------------------------+------------------------+-------+ Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. These -may not be available on all platforms when used with the :meth:`strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. +convenience. These parameters all correspond to ISO 8601 date values. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | @@ -2194,16 +2412,59 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. | ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | | | number where 1 is Monday. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | | | the first day of the week. | | | | | Week 01 is the week containing | | | | | Jan 4. | | | +-----------+--------------------------------+------------------------+-------+ +These may not be available on all platforms when used with the :meth:`strftime` +method. The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. + +The full set of format codes supported varies across platforms, because Python +calls the platform C library's :func:`strftime` function, and platform +variations are common. To see the full set of format codes supported on your +platform, consult the :manpage:`strftime(3)` documentation. + .. versionadded:: 3.6 ``%G``, ``%u`` and ``%V`` were added. +Technical Detail +^^^^^^^^^^^^^^^^ + +Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's +``time.strftime(fmt, d.timetuple())`` although not all objects support a +:meth:`timetuple` method. + +For the :meth:`datetime.strptime` class method, the default value is +``1900-01-01T00:00:00.000``: any components not specified in the format string +will be pulled from the default value. [#]_ + +Using ``datetime.strptime(date_string, format)`` is equivalent to:: + + datetime(*(time.strptime(date_string, format)[0:6])) + +except when the format includes sub-second components or timezone offset +information, which are supported in ``datetime.strptime`` but are discarded by +``time.strptime``. + +For :class:`.time` objects, the format codes for year, month, and day should not +be used, as :class:`time` objects have no such values. If they're used anyway, +``1900`` is substituted for the year, and ``1`` for the month and day. + +For :class:`date` objects, the format codes for hours, minutes, seconds, and +microseconds should not be used, as :class:`date` objects have no such +values. If they're used anyway, ``0`` is substituted for them. + +For the same reason, handling of format strings containing Unicode code points +that can't be represented in the charset of the current locale is also +platform-dependent. On some platforms such code points are preserved intact in +the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return +an empty string instead. + Notes: (1) @@ -2237,7 +2498,7 @@ Notes: (5) When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is + accepts from one to six digits and zero pads on the right. ``%f`` is an extension to the set of format characters in the C standard (but implemented separately in datetime objects, and therefore always available). @@ -2250,13 +2511,13 @@ Notes: ``%z`` :meth:`utcoffset` is transformed into a string of the form - ±HHMM[SS[.ffffff]], where HH is a 2-digit string giving the number of UTC - offset hours, MM is a 2-digit string giving the number of UTC offset - minutes, SS is a 2-digit string giving the number of UTC offset - seconds and ffffff is a 6-digit string giving the number of UTC - offset microseconds. The ffffff part is omitted when the offset is a - whole number of seconds and both the ffffff and the SS part is omitted - when the offset is a whole number of minutes. For example, if + ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number + of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC + offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset + seconds and ``ffffff`` is a 6-digit string giving the number of UTC + offset microseconds. The ``ffffff`` part is omitted when the offset is a + whole number of seconds and both the ``ffffff`` and the ``SS`` part is + omitted when the offset is a whole number of minutes. For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string ``'-0330'``. @@ -2272,12 +2533,12 @@ Notes: ``%Z`` If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty - string. Otherwise ``%Z`` is replaced by the returned value, which must + string. Otherwise ``%Z`` is replaced by the returned value, which must be a string. .. versionchanged:: 3.2 When the ``%z`` directive is provided to the :meth:`strptime` method, an - aware :class:`.datetime` object will be produced. The ``tzinfo`` of the + aware :class:`.datetime` object will be produced. The ``tzinfo`` of the result will be set to a :class:`timezone` instance. (7) @@ -2291,7 +2552,23 @@ Notes: :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not interchangeable. +(9) + When used with the :meth:`strptime` method, the leading zero is optional + for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%J``, ``%U``, + ``%W``, and ``%V``. Format ``%y`` does require a leading zero. + .. rubric:: Footnotes .. [#] If, that is, we ignore the effects of Relativity + +.. [#] This matches the definition of the "proleptic Gregorian" calendar in + Dershowitz and Reingold's book *Calendrical Calculations*, + where it's the base calendar for all computations. See the book for + algorithms for converting between proleptic Gregorian ordinals and + many other calendar systems. + +.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar + `_ + for a good explanation. + .. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 88a2fa62a56588..60223657a44043 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -5,10 +5,13 @@ Debugging and Profiling These libraries help you with Python development: the debugger enables you to step through code, analyze stack frames and set breakpoints etc., and the profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. +allowing you to identify bottlenecks in your programs. Auditing events +provide visibility into runtime behaviors that would otherwise require +intrusive debugging or patching. .. toctree:: + audit_events.rst bdb.rst faulthandler.rst pdb.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index bcae55eb821784..38173f18ca6602 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1357,6 +1357,9 @@ In addition to the three supplied contexts, new contexts can be created with the The rounding mode of the context is used. Results are always correctly-rounded in the Python version. + ``Decimal(0) ** Decimal(0)`` results in ``InvalidOperation``, and if ``InvalidOperation`` + is not trapped, then results in ``Decimal('NaN')``. + .. versionchanged:: 3.3 The C module computes :meth:`power` in terms of the correctly-rounded :meth:`exp` and :meth:`ln` functions. The result is well-defined but @@ -1475,9 +1478,18 @@ are also included in the pure Python version for compatibility. .. data:: HAVE_THREADS - The default value is ``True``. If Python is compiled without threads, the - C version automatically disables the expensive thread local context - machinery. In this case, the value is ``False``. + The value is ``True``. Deprecated, because Python now always has threads. + +.. deprecated:: 3.9 + +.. data:: HAVE_CONTEXTVAR + + The default value is ``True``. If Python is compiled ``--without-decimal-contextvar``, + the C version uses a thread-local rather than a coroutine-local context and the value + is ``False``. This is slightly faster in some nested context scenarios. + +.. versionadded:: 3.9 backported to 3.7 and 3.8 + Rounding modes -------------- diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index f044cb2d6e0a86..a8543b38c197ec 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -18,12 +18,13 @@ -------------- This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce difference -information in various formats, including HTML and context and unified +can be used for example, for comparing files, and can produce information +about file differences in various formats, including HTML and context and unified diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. class:: SequenceMatcher + :noindex: This is a flexible class for comparing pairs of sequences of any type, so long as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a @@ -127,6 +128,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. the next difference highlight at the top of the browser without any leading context). + .. note:: + *fromdesc* and *todesc* are interpreted as unescaped HTML and should be + properly escaped while receiving input from untrusted sources. + .. versionchanged:: 3.5 *charset* keyword-only argument was added. The default charset of HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``. @@ -334,14 +339,14 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. function:: IS_LINE_JUNK(line) - Return true for ignorable lines. The line *line* is ignorable if *line* is + Return ``True`` for ignorable lines. The line *line* is ignorable if *line* is blank or contains a single ``'#'``, otherwise it is not ignorable. Used as a default for parameter *linejunk* in :func:`ndiff` in older versions. .. function:: IS_CHARACTER_JUNK(ch) - Return true for ignorable characters. The character *ch* is ignorable if *ch* + Return ``True`` for ignorable characters. The character *ch* is ignorable if *ch* is a space or tab, otherwise it is not ignorable. Used as a default for parameter *charjunk* in :func:`ndiff`. @@ -366,7 +371,7 @@ The :class:`SequenceMatcher` class has this constructor: Optional argument *isjunk* must be ``None`` (the default) or a one-argument function that takes a sequence element and returns true if and only if the element is "junk" and should be ignored. Passing ``None`` for *isjunk* is - equivalent to passing ``lambda x: 0``; in other words, no elements are ignored. + equivalent to passing ``lambda x: False``; in other words, no elements are ignored. For example, pass:: lambda x: x in " \t" @@ -543,6 +548,16 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. + .. note:: + + Caution: The result of a :meth:`ratio` call may depend on the order of + the arguments. For instance:: + + >>> SequenceMatcher(None, 'tide', 'diet').ratio() + 0.25 + >>> SequenceMatcher(None, 'diet', 'tide').ratio() + 0.5 + .. method:: quick_ratio() @@ -634,6 +649,7 @@ The :class:`Differ` class has this constructor: .. class:: Differ(linejunk=None, charjunk=None) + :noindex: Optional keyword parameters *linejunk* and *charjunk* are for filter functions (or ``None``): diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 2a3ffb5e827119..f282415cc33d48 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -244,7 +244,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: findlabels(code) - Detect all offsets in the code object *code* which are jump targets, and + Detect all offsets in the raw compiled bytecode string *code* which are jump targets, and return a list of these offsets. @@ -346,7 +346,7 @@ The Python compiler currently generates the following bytecode instructions. .. opcode:: ROT_FOUR - Lifts second, third and forth stack items one position up, moves top down + Lifts second, third and fourth stack items one position up, moves top down to position four. .. versionadded:: 3.8 @@ -640,15 +640,17 @@ the original TOS1. .. opcode:: LIST_APPEND (i) - Calls ``list.append(TOS[-i], TOS)``. Used to implement list comprehensions. + Calls ``list.append(TOS1[-i], TOS)``. Used to implement list comprehensions. .. opcode:: MAP_ADD (i) - Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict + Calls ``dict.__setitem__(TOS1[-i], TOS1, TOS)``. Used to implement dict comprehensions. .. versionadded:: 3.1 + .. versionchanged:: 3.8 + Map value is TOS and map key is TOS1. Before, those were reversed. For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the @@ -892,9 +894,9 @@ All of the following opcodes use their arguments. .. opcode:: BUILD_CONST_KEY_MAP (count) - The version of :opcode:`BUILD_MAP` specialized for constant keys. *count* - values are consumed from the stack. The top element on the stack contains - a tuple of keys. + The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the + top element on the stack which contains a tuple of keys, then starting from + ``TOS1``, pops *count* values to form values in the built dictionary. .. versionadded:: 3.6 @@ -1174,27 +1176,29 @@ All of the following opcodes use their arguments. .. opcode:: LOAD_METHOD (namei) - Loads a method named ``co_names[namei]`` from TOS object. TOS is popped and - method and TOS are pushed when interpreter can call unbound method directly. - TOS will be used as the first argument (``self``) by :opcode:`CALL_METHOD`. - Otherwise, ``NULL`` and method is pushed (method is bound method or - something else). + Loads a method named ``co_names[namei]`` from the TOS object. TOS is popped. + This bytecode distinguishes two cases: if TOS has a method with the correct + name, the bytecode pushes the unbound method and TOS. TOS will be used as + the first argument (``self``) by :opcode:`CALL_METHOD` when calling the + unbound method. Otherwise, ``NULL`` and the object return by the attribute + lookup are pushed. .. versionadded:: 3.7 .. opcode:: CALL_METHOD (argc) - Calls a method. *argc* is number of positional arguments. + Calls a method. *argc* is the number of positional arguments. Keyword arguments are not supported. This opcode is designed to be used with :opcode:`LOAD_METHOD`. Positional arguments are on top of the stack. - Below them, two items described in :opcode:`LOAD_METHOD` on the stack. - All of them are popped and return value is pushed. + Below them, the two items described in :opcode:`LOAD_METHOD` are on the + stack (either ``self`` and an unbound method object or ``NULL`` and an + arbitrary callable). All of them are popped and the return value is pushed. .. versionadded:: 3.7 -.. opcode:: MAKE_FUNCTION (argc) +.. opcode:: MAKE_FUNCTION (flags) Pushes a new function object on the stack. From bottom to top, the consumed stack must consist of values if the argument carries a specified flag value @@ -1219,10 +1223,10 @@ All of the following opcodes use their arguments. .. opcode:: EXTENDED_ARG (ext) - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + Prefixes any opcode which has an argument too big to fit into the default one + byte. *ext* holds an additional byte which act as higher bits in the argument. + For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming + an argument from two-byte to four-byte. .. opcode:: FORMAT_VALUE (flags) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index e7c0033eb6bc45..a77322f83acbde 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1,5 +1,3 @@ -:keepdoctest: - :mod:`doctest` --- Test interactive Python examples =================================================== @@ -1531,7 +1529,7 @@ OutputChecker objects A class used to check the whether the actual output from a doctest example matches the expected output. :class:`OutputChecker` defines two methods: - :meth:`check_output`, which compares a given pair of outputs, and returns true + :meth:`check_output`, which compares a given pair of outputs, and returns ``True`` if they match; and :meth:`output_difference`, which returns a string describing the differences between two outputs. diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 09ea64a5a01aae..c68e773b1688aa 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -23,7 +23,7 @@ policy :attr:`~email.policy.Compat32`. If you are going to use another policy, you should be using the :class:`~email.message.EmailMessage` class instead. An email message consists of *headers* and a *payload*. Headers must be -:rfc:`5233` style names and values, where the field name and value are +:rfc:`5322` style names and values, where the field name and value are separated by a colon. The colon is not part of either the field name or the field value. The payload may be a simple text message, or a binary object, or a structured sequence of sub-messages each with their own set of headers and @@ -308,7 +308,7 @@ Here are the methods of the :class:`Message` class: .. method:: __contains__(name) - Return true if the message object has a field named *name*. Matching is + Return ``True`` if the message object has a field named *name*. Matching is done case-insensitively and *name* should not include the trailing colon. Used for the ``in`` operator, e.g.:: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index e09c7c0e402bbc..918fc55677e723 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -116,7 +116,7 @@ Currently the email package provides only one concrete content manager, decoding the payload to unicode. The default error handler is ``replace``. - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ + .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8', \ cte=None, \ disposition=None, filename=None, cid=None, \ params=None, headers=None) diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 511ad163583197..f4b9f52509689e 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -99,7 +99,7 @@ All defect classes are subclassed from :class:`email.errors.MessageDefect`. * :class:`MultipartInvariantViolationDefect` -- A message claimed to be a :mimetype:`multipart`, but no subparts were found. Note that when a message has this defect, its :meth:`~email.message.Message.is_multipart` method may - return false even though its content type claims to be :mimetype:`multipart`. + return ``False`` even though its content type claims to be :mimetype:`multipart`. * :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 encoded bytes, the padding was not correct. Enough padding is added to diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 9376da2b8d39ce..3e1d97a03264b2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -289,7 +289,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A :class:`ParameterizedMIMEHeader` class that handles the :mailheader:`Content-Disposition` header. - .. attribute:: content-disposition + .. attribute:: content_disposition ``inline`` and ``attachment`` are the only valid values in common use. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index f1806a0866bb81..5e0509f4181199 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -178,7 +178,7 @@ message objects. .. method:: __contains__(name) - Return true if the message object has a field named *name*. Matching is + Return ``True`` if the message object has a field named *name*. Matching is done without regard to case and *name* does not include the trailing colon. Used for the ``in`` operator. For example:: diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8e7076259810f5..bf53b9520fc723 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -210,7 +210,7 @@ added matters. To illustrate:: :meth:`register_defect` method. - .. attribute:: mangle_from\_ + .. attribute:: mangle_from_ If :const:`True`, lines starting with *"From "* in the body are escaped by putting a ``>`` in front of them. This parameter is used when diff --git a/Doc/library/email.rst b/Doc/library/email.rst index fae99cf3e6abbe..5eebcd9e896d93 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -16,7 +16,7 @@ The :mod:`email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as :mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as -RFC-compliant as possible, supporting :rfc:`5233` and :rfc:`6532`, as well as +RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 63fae2ab84e213..4d0e920eb0ad29 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -117,8 +117,8 @@ of the new API. a 10-tuple; the first 9 elements make up a tuple that can be passed directly to :func:`time.mktime`, and the tenth is the offset of the date's timezone from UTC (which is the official term for Greenwich Mean Time) [#]_. If the input string - has no timezone, the last element of the tuple returned is ``None``. Note that - indexes 6, 7, and 8 of the result tuple are not usable. + has no timezone, the last element of the tuple returned is ``0``, which represents + UTC. Note that indexes 6, 7, and 8 of the result tuple are not usable. .. function:: parsedate_to_datetime(date) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index c797f63326d1a2..a5221250c40486 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -74,7 +74,7 @@ options: script will *not* be installed. * ``--default-pip``: if a "default pip" installation is requested, the - ``pip`` script will be installed in addition to the two regular scripts. + ``pip`` script will be installed in addition to the two regular scripts. Providing both of the script selection options will trigger an exception. @@ -119,6 +119,8 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap + .. note:: The bootstrapping process has side effects on both ``sys.path`` and diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 19277d76995fed..40499eb93a1ad0 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -19,6 +19,12 @@ An enumeration is a set of symbolic names (members) bound to unique, constant values. Within an enumeration, the members can be compared by identity, and the enumeration itself can be iterated over. +.. note:: Case of Enum Members + + Because Enums are used to represent constants we recommend using + UPPER_CASE names for enum members, and will be using that style + in our examples. + Module Contents --------------- @@ -50,12 +56,13 @@ helper, :class:`auto`. the bitwise operations without losing their :class:`Flag` membership. .. function:: unique + :noindex: Enum class decorator that ensures only one name is bound to any one value. .. class:: auto - Instances are replaced with an appropriate value for Enum members. + Instances are replaced with an appropriate value for Enum members. Initial value starts at 1. .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` @@ -269,10 +276,14 @@ overridden:: .. note:: - The goal of the default :meth:`_generate_next_value_` methods is to provide + The goal of the default :meth:`_generate_next_value_` method is to provide the next :class:`int` in sequence with the last :class:`int` provided, but the way it does this is an implementation detail and may change. +.. note:: + + The :meth:`_generate_next_value_` method must be defined before any members. + Iteration --------- @@ -739,9 +750,11 @@ Some rules: :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. 5. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the mixed-in - type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or - :func:`repr` is desired, use the `!s` or `!r` format codes. + and :func:`format` will use the mixed-in type's :meth:`__format__` + unless :meth:`__str__` or :meth:`__format__` is overridden in the subclass, + in which case the overridden methods or :class:`Enum` methods will be used. + Use the !s and !r format codes to force usage of the :class:`Enum` class's + :meth:`__str__` and :meth:`__repr__` methods. When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------ @@ -875,6 +888,32 @@ Using an auto-numbering :meth:`__new__` would look like:: >>> Color.GREEN.value 2 +To make a more general purpose ``AutoNumber``, add ``*args`` to the signature:: + + >>> class AutoNumber(NoValue): + ... def __new__(cls, *args): # this is the only change from above + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + +Then when you inherit from ``AutoNumber`` you can write your own ``__init__`` +to handle any extra arguments:: + + >>> class Swatch(AutoNumber): + ... def __init__(self, pantone='unknown'): + ... self.pantone = pantone + ... AUBURN = '3497' + ... SEA_GREEN = '1246' + ... BLEACHED_CORAL = () # New color, no Pantone code yet! + ... + >>> Swatch.SEA_GREEN + + >>> Swatch.SEA_GREEN.pantone + '1246' + >>> Swatch.BLEACHED_CORAL.pantone + 'unknown' .. note:: diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 52a505e0a0ff8f..368ae45e5815b1 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -42,12 +42,12 @@ include the originating exception(s) and the final exception. When raising a new exception (rather than using a bare ``raise`` to re-raise the exception currently being handled), the implicit exception context can be -supplemented with an explicit cause by using :keyword:`from` with +supplemented with an explicit cause by using :keyword:`from` with :keyword:`raise`:: raise new_exc from original_exc -The expression following :keyword:`from` must be an exception or ``None``. It +The expression following :keyword:`from` must be an exception or ``None``. It will be set as :attr:`__cause__` on the raised exception. Setting :attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` attribute to ``True``, so that using ``raise new_exc from None`` @@ -397,9 +397,25 @@ The following exceptions are the exceptions that are usually raised. or :func:`eval`, or when reading the initial script or standard input (also interactively). - Instances of this class have attributes :attr:`filename`, :attr:`lineno`, - :attr:`offset` and :attr:`text` for easier access to the details. :func:`str` - of the exception instance returns only the message. + The :func:`str` of the exception instance returns only the error message. + + .. attribute:: filename + + The name of the file the syntax error occurred in. + + .. attribute:: lineno + + Which line number in the file the error occurred in. This is + 1-indexed: the first line in the file has a ``lineno`` of 1. + + .. attribute:: offset + + The column in the line where the error occurred. This is + 1-indexed: the first character in the line has an ``offset`` of 1. + + .. attribute:: text + + The source code text involved in the error. .. exception:: IndentationError diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 94ebd87639c501..b588dfa18db2dd 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -100,8 +100,10 @@ Dumping the tracebacks after a timeout :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file descriptors `. - This function is implemented using a watchdog thread and therefore is not - available if Python is compiled with threads disabled. + This function is implemented using a watchdog thread. + + .. versionchanged:: 3.7 + This function is now always available. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 2db9674952d7b6..69484b647363dd 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -57,6 +57,8 @@ The module defines the following functions: If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl + .. function:: ioctl(fd, request, arg=0, mutate_flag=True) @@ -106,6 +108,8 @@ The module defines the following functions: >>> buf array('h', [13341]) + .. audit-event:: fcntl.ioctl fd,request,arg fcntl.ioctl + .. function:: flock(fd, operation) @@ -116,11 +120,14 @@ The module defines the following functions: If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + .. audit-event:: fcntl.flock fd,operation fcntl.flock + .. function:: lockf(fd, cmd, len=0, start=0, whence=0) This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. - *fd* is the file descriptor of the file to lock or unlock, and *cmd* + *fd* is the file descriptor (file objects providing a :meth:`~io.IOBase.fileno` + method are accepted as well) of the file to lock or unlock, and *cmd* is one of the following values: * :const:`LOCK_UN` -- unlock @@ -148,6 +155,8 @@ The module defines the following functions: The default for *len* is 0 which means to lock to the end of the file. The default for *whence* is also 0. + .. audit-event:: fcntl.lockf fd,cmd,len,start,whence fcntl.lockf + Examples (all on a SVR4 compliant system):: import struct, fcntl, os @@ -171,4 +180,3 @@ using the :func:`flock` call may be better. present in the :mod:`os` module (on BSD only), the :func:`os.open` function provides an alternative to the :func:`lockf` and :func:`flock` functions. - diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index f5e5280a136399..cc4039a30e38ae 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -109,14 +109,14 @@ if there is no active state, :exc:`RuntimeError` is raised. .. function:: isfirstline() - Returns true if the line just read is the first line of its file, otherwise - returns false. + Return ``True`` if the line just read is the first line of its file, otherwise + return ``False``. .. function:: isstdin() - Returns true if the last line was read from ``sys.stdin``, otherwise returns - false. + Return ``True`` if the last line was read from ``sys.stdin``, otherwise return + ``False``. .. function:: nextfile() diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ce07d326b395d8..925f08e914685e 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -75,7 +75,7 @@ patterns. .. function:: filter(names, pattern) - Return the subset of the list of *names* that match *pattern*. It is the same as + Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index b5a818e1cafa61..58e7126b0bf212 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -94,6 +94,13 @@ another rational number, or from a string. Denominator of the Fraction in lowest term. + .. method:: as_integer_ratio() + + Return a tuple of two integers, whose ratio is equal + to the Fraction and with a positive denominator. + + .. versionadded:: 3.8 + .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 6c39f9a59fc1fd..db0212367340d9 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -33,7 +33,8 @@ Here's a sample session using the :mod:`ftplib` module:: drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools '226 Directory send OK.' - >>> ftp.retrbinary('RETR README', open('README', 'wb').write) + >>> with open('README', 'wb') as fp: + >>> ftp.retrbinary('RETR README', fp.write) '226 Transfer complete.' >>> ftp.quit() @@ -144,7 +145,7 @@ The module defines the following items: The set of all exceptions (as a tuple) that methods of :class:`FTP` instances may raise as a result of problems with the FTP connection (as opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError`. + four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. .. seealso:: @@ -190,6 +191,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect + .. versionchanged:: 3.3 *source_address* parameter was added. @@ -223,6 +226,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Send a simple command string to the server and return the response string. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd + .. method:: FTP.voidcmd(cmd) @@ -230,6 +235,8 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. nothing if a response code corresponding to success (codes in the range 200--299) is received. Raise :exc:`error_reply` otherwise. + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd + .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 415a65b4946f08..bc0285e2582a51 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -128,6 +128,8 @@ are always available. They are listed here in alphabetical order. :func:`breakpoint` will automatically call that, allowing you to drop into the debugger of choice. + .. audit-event:: builtins.breakpoint breakpointhook breakpoint + .. versionadded:: 3.7 .. _func-bytearray: @@ -149,8 +151,8 @@ are always available. They are listed here in alphabetical order. * If it is an *integer*, the array will have that size and will be initialized with null bytes. - * If it is an object conforming to the *buffer* interface, a read-only buffer - of the object will be used to initialize the bytes array. + * If it is an object conforming to the :ref:`buffer interface `, + a read-only buffer of the object will be used to initialize the bytes array. * If it is an *iterable*, it must be an iterable of integers in the range ``0 <= x < 256``, which are used as the initial contents of the array. @@ -179,8 +181,8 @@ are always available. They are listed here in alphabetical order. .. function:: callable(object) Return :const:`True` if the *object* argument appears callable, - :const:`False` if not. If this returns true, it is still possible that a - call fails, but if it is false, calling *object* will never succeed. + :const:`False` if not. If this returns ``True``, it is still possible that a + call fails, but if it is ``False``, calling *object* will never succeed. Note that classes are callable (calling a class returns a new instance); instances are callable if their class has a :meth:`__call__` method. @@ -275,9 +277,9 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. - .. audit-event:: compile "source filename" + .. audit-event:: compile source,filename compile - Raises an :func:`auditing event ` ``compile`` with arguments + Raises an :ref:`auditing event ` ``compile`` with arguments ``source`` and ``filename``. This event may also be raised by implicit compilation. @@ -452,7 +454,7 @@ are always available. They are listed here in alphabetical order. n += 1 -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(expression[, globals[, locals]]) The arguments are a string and optional globals and locals. If provided, *globals* must be a dictionary. If provided, *locals* can be any mapping @@ -463,12 +465,16 @@ are always available. They are listed here in alphabetical order. dictionaries as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. - This means that *expression* normally has full - access to the standard :mod:`builtins` module and restricted environments are - propagated. If the *locals* dictionary is omitted it defaults to the *globals* - dictionary. If both dictionaries are omitted, the expression is executed in the - environment where :func:`eval` is called. The return value is the result of + inserted under that key before *expression* is parsed. This means that + *expression* normally has full access to the standard :mod:`builtins` + module and restricted environments are propagated. If the *locals* + dictionary is omitted it defaults to the *globals* dictionary. If both + dictionaries are omitted, the expression is executed with the *globals* and + *locals* in the environment where :func:`eval` is called. Note, *eval()* + does not have access to the :term:`nested scopes ` (non-locals) in the + enclosing environment. + + The return value is the result of the evaluated expression. Syntax errors are reported as exceptions. Example: >>> x = 1 @@ -488,10 +494,10 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. - .. audit-event:: exec code_object + .. audit-event:: exec code_object eval - Raises an :func:`auditing event ` ``exec`` with the code object as - the argument. Code compilation events may also be raised. + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. .. index:: builtin: exec @@ -503,7 +509,8 @@ are always available. They are listed here in alphabetical order. occurs). [#]_ If it is a code object, it is simply executed. In all cases, the code that's executed is expected to be valid as file input (see the section "File input" in the Reference Manual). Be aware that the - :keyword:`return` and :keyword:`yield` statements may not be used outside of + :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` + statements may not be used outside of function definitions even within the context of code passed to the :func:`exec` function. The return value is ``None``. @@ -523,10 +530,10 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. - .. audit-event:: exec code_object + .. audit-event:: exec code_object exec - Raises an :func:`auditing event ` ``exec`` with the code object as - the argument. Code compilation events may also be raised. + Raises an :ref:`auditing event ` ``exec`` with the code object + as the argument. Code compilation events may also be raised. .. note:: @@ -575,7 +582,7 @@ are always available. They are listed here in alphabetical order. input must conform to the following grammar after leading and trailing whitespace characters are removed: - .. productionlist:: + .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" @@ -761,6 +768,8 @@ are always available. They are listed here in alphabetical order. .. impl-detail:: This is the address of the object in memory. + .. audit-event:: builtins.id id id + .. function:: input([prompt]) @@ -777,12 +786,12 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. - .. audit-event:: builtins.input prompt + .. audit-event:: builtins.input prompt input - Raises an :func:`auditing event ` ``builtins.input`` with + Raises an :ref:`auditing event ` ``builtins.input`` with argument ``prompt`` before reading input - .. audit-event:: builtins.input/result result + .. audit-event:: builtins.input/result result input Raises an auditing event ``builtins.input/result`` with the result after successfully reading input. @@ -829,22 +838,30 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 Falls back to :meth:`__index__` if :meth:`__int__` is not defined. + .. versionchanged:: 3.8.14 + :class:`int` string inputs and string representations can be limited to + help avoid denial of service attacks. A :exc:`ValueError` is raised when + the limit is exceeded while converting a string *x* to an :class:`int` or + when converting an :class:`int` into a string would exceed the limit. + See the :ref:`integer string conversion length limitation + ` documentation. + .. function:: isinstance(object, classinfo) - Return true if the *object* argument is an instance of the *classinfo* + Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect or :term:`virtual `) subclass thereof. If *object* is not - an object of the given type, the function always returns false. + an object of the given type, the function always returns ``False``. If *classinfo* is a tuple of type objects (or recursively, other such - tuples), return true if *object* is an instance of any of the types. + tuples), return ``True`` if *object* is an instance of any of the types. If *classinfo* is not a type or tuple of types and such tuples, a :exc:`TypeError` exception is raised. .. function:: issubclass(class, classinfo) - Return true if *class* is a subclass (direct, indirect or :term:`virtual + Return ``True`` if *class* is a subclass (direct, indirect or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects, in which case every entry in *classinfo* will be checked. In any other @@ -944,7 +961,7 @@ are always available. They are listed here in alphabetical order. .. _func-memoryview: -.. function:: memoryview(obj) +.. class:: memoryview(obj) :noindex: Return a "memory view" object created from the given argument. See @@ -1029,7 +1046,8 @@ are always available. They are listed here in alphabetical order. .. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. + cannot be opened, an :exc:`OSError` is raised. See + :ref:`tut-files` for more examples of how to use this function. *file* is a :term:`path-like object` giving the pathname (absolute or relative to the current working directory) of the file to be opened or an @@ -1062,12 +1080,12 @@ are always available. They are listed here in alphabetical order. ``'a'`` open for writing, appending to the end of the file if it exists ``'b'`` binary mode ``'t'`` text mode (default) - ``'+'`` open a disk file for updating (reading and writing) + ``'+'`` open for updating (reading and writing) ========= =============================================================== The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). - For binary read-write access, the mode ``'w+b'`` opens and truncates the file - to 0 bytes. ``'r+b'`` opens the file without truncation. + Modes ``'w+'`` and ``'w+b'`` open and truncate the file. Modes ``'r+'`` + and ``'r+b'`` open the file with no truncation. As mentioned in the :ref:`io-overview`, Python distinguishes between binary and text I/O. Files opened in binary mode (including ``'b'`` in the *mode* @@ -1220,7 +1238,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open "file mode flags" + .. audit-event:: open file,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. @@ -1239,7 +1257,7 @@ are always available. They are listed here in alphabetical order. * The file is now non-inheritable. - .. deprecated-removed:: 3.4 4.0 + .. deprecated-removed:: 3.4 3.9 The ``'U'`` mode. @@ -1266,11 +1284,12 @@ are always available. They are listed here in alphabetical order. returns ``8364``. This is the inverse of :func:`chr`. -.. function:: pow(x, y[, z]) +.. function:: pow(base, exp[, mod]) - Return *x* to the power *y*; if *z* is present, return *x* to the power *y*, - modulo *z* (computed more efficiently than ``pow(x, y) % z``). The two-argument - form ``pow(x, y)`` is equivalent to using the power operator: ``x**y``. + Return *base* to the power *exp*; if *mod* is present, return *base* to the + power *exp*, modulo *mod* (computed more efficiently than + ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is + equivalent to using the power operator: ``base**exp``. The arguments must have numeric types. With mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` @@ -1279,14 +1298,15 @@ are always available. They are listed here in alphabetical order. converted to float and a float result is delivered. For example, ``10**2`` returns ``100``, but ``10**-2`` returns ``0.01``. - For :class:`int` operands *x* and *y*, if *z* is present, *z* must also be - of integer type and *z* must be nonzero. If *z* is present and *y* is - negative, *x* must be relatively prime to *z*. In that case, ``pow(inv_x, - -y, z)`` is returned, where *inv_x* is an inverse to *x* modulo *z*. + For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must + also be of integer type and *mod* must be nonzero. If *mod* is present and + *exp* is negative, *base* must be relatively prime to *mod*. In that case, + ``pow(inv_base, -exp, mod)`` is returned, where *inv_base* is an inverse to + *base* modulo *mod*. Here's an example of computing an inverse for ``38`` modulo ``97``:: - >>> pow(38, -1, 97) + >>> pow(38, -1, mod=97) 23 >>> 23 * 38 % 97 == 1 True @@ -1296,6 +1316,10 @@ are always available. They are listed here in alphabetical order. the second argument to be negative, permitting computation of modular inverses. + .. versionchanged:: 3.8 + Allow keyword arguments. Formerly, only positional arguments were + supported. + .. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -1400,7 +1424,7 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. function:: range(stop) +.. class:: range(stop) range(start, stop[, step]) :noindex: @@ -1562,11 +1586,11 @@ are always available. They are listed here in alphabetical order. about strings, see :ref:`textseq`. -.. function:: sum(iterable[, start]) +.. function:: sum(iterable, /, start=0) Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. + total. The *iterable*'s items are normally numbers, and the start value is not + allowed to be a string. For some use cases, there are good alternatives to :func:`sum`. The preferred, fast way to concatenate a sequence of strings is by calling @@ -1581,10 +1605,17 @@ are always available. They are listed here in alphabetical order. Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have - been overridden in a class. The search order is same as that used by - :func:`getattr` except that the *type* itself is skipped. + been overridden in a class. - The :attr:`~class.__mro__` attribute of the *type* lists the method + The *object-or-type* determines the :term:`method resolution order` + to be searched. The search starts from the class right after the + *type*. + + For example, if :attr:`~class.__mro__` of *object-or-type* is + ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, + then :func:`super` searches ``C -> A -> object``. + + The :attr:`~class.__mro__` attribute of the *object-or-type* lists the method resolution search order used by both :func:`getattr` and :func:`super`. The attribute is dynamic and can change whenever the inheritance hierarchy is updated. @@ -1604,7 +1635,7 @@ are always available. They are listed here in alphabetical order. not found in statically compiled languages or languages that only support single inheritance. This makes it possible to implement "diamond diagrams" where multiple base classes implement the same method. Good design dictates - that this method have the same calling signature in every case (because the + that such implementations have the same calling signature in every case (because the order of calls is determined at runtime, because that order adapts to changes in the class hierarchy, and because that order can include sibling classes that are unknown prior to runtime). @@ -1616,6 +1647,10 @@ are always available. They are listed here in alphabetical order. super().method(arg) # This does the same thing as: # super(C, self).method(arg) + In addition to method lookups, :func:`super` also works for attribute + lookups. One possible use case for this is calling :term:`descriptors ` + in a parent or sibling class. + Note that :func:`super` is implemented as part of the binding process for explicit dotted attribute lookups such as ``super().__getitem__(name)``. It does so by implementing its own :meth:`__getattribute__` method for searching @@ -1636,7 +1671,7 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. function:: tuple([iterable]) +.. class:: tuple([iterable]) :noindex: Rather than being a function, :class:`tuple` is actually an immutable @@ -1644,7 +1679,7 @@ are always available. They are listed here in alphabetical order. .. class:: type(object) - type(name, bases, dict) + type(name, bases, dict, **kwds) .. index:: object: type @@ -1657,21 +1692,29 @@ are always available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~definition.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and is copied to a standard dictionary to become the - :attr:`~object.__dict__` attribute. For example, the following two - statements create identical :class:`type` objects: + dynamic form of the :keyword:`class` statement. The *name* string is + the class name and becomes the :attr:`~definition.__name__` attribute. + The *bases* tuple contains the base classes and becomes the + :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + ultimate base of all classes, is added. The *dict* dictionary contains + attribute and method definitions for the class body; it may be copied + or wrapped before becoming the :attr:`~object.__dict__` attribute. + The following two statements create identical :class:`type` objects: >>> class X: ... a = 1 ... - >>> X = type('X', (object,), dict(a=1)) + >>> X = type('X', (), dict(a=1)) See also :ref:`bltin-type-objects`. + Keyword arguments provided to the three argument form are passed to the + appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) + in the same way that keywords in a class + definition (besides *metaclass*) would. + + See also :ref:`class-customization`. + .. versionchanged:: 3.6 Subclasses of :class:`type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. @@ -1690,6 +1733,9 @@ are always available. They are listed here in alphabetical order. locals dictionary is only useful for reads since updates to the locals dictionary are ignored. + A :exc:`TypeError` exception is raised if an object is specified but + it doesn't have a :attr:`~object.__dict__` attribute (for example, if + its class defines the :attr:`~object.__slots__` attribute). .. function:: zip(*iterables) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3a0b554e923c7a..0fb8d900c73627 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -101,8 +101,7 @@ The :mod:`functools` module defines the following functions: return sum(sentence.count(vowel) for vowel in 'aeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. + grow without bound. If *typed* is set to true, function arguments of different types will be cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 084cd6ac257ebe..dcbfe7f1d9db5d 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -35,7 +35,7 @@ The :mod:`gc` module provides the following functions: .. function:: isenabled() - Returns true if automatic collection is enabled. + Return ``True`` if automatic collection is enabled. .. function:: collect(generation=2) @@ -72,6 +72,8 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.8 New *generation* parameter. + .. audit-event:: gc.get_objects generation gc.get_objects + .. function:: get_stats() Return a list of three per-generation dictionaries containing collection @@ -106,9 +108,9 @@ The :mod:`gc` module provides the following functions: allocations minus the number of deallocations exceeds *threshold0*, collection starts. Initially only generation ``0`` is examined. If generation ``0`` has been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. Similarly, *threshold2* - controls the number of collections of generation ``1`` before collecting - generation ``2``. + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. .. function:: get_count() @@ -135,10 +137,13 @@ The :mod:`gc` module provides the following functions: resulting referrers. To get only currently live objects, call :func:`collect` before calling :func:`get_referrers`. - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + .. audit-event:: gc.get_referrers objs gc.get_referrers .. function:: get_referents(*objs) @@ -151,6 +156,7 @@ The :mod:`gc` module provides the following functions: be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. + .. audit-event:: gc.get_referents objs gc.get_referents .. function:: is_tracked(obj) @@ -212,7 +218,7 @@ values but should not rebind them): A list of objects which the collector found to be unreachable but could not be freed (uncollectable objects). Starting with Python 3.4, this list should be empty most of the time, except when using instances of - C extension types with a non-NULL ``tp_del`` slot. + C extension types with a non-``NULL`` ``tp_del`` slot. If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be added to this list rather than freed. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 937330bb201b08..ec2c12806b4160 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -724,8 +724,8 @@ implementations, and valuable experience to the creation of this module: .. [#] The default locale directory is system dependent; for example, on RedHat Linux it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. The :mod:`gettext` module does not try to support these system dependent - defaults; instead its default is :file:`{sys.prefix}/share/locale` (see - :data:`sys.prefix`). For this reason, it is always best to call + defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see + :data:`sys.base_prefix`). For this reason, it is always best to call :func:`bindtextdomain` with an explicit absolute path at the start of your application. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 2a5f0ddc4ef319..3c468ebf73769f 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -43,14 +43,19 @@ For example, ``'[?]'`` matches the character ``'?'``. (like :file:`/usr/src/Python-1.5/Makefile`) or relative (like :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken symlinks are included in the results (as in the shell). Whether or not the - results are sorted depends on the file system. + results are sorted depends on the file system. If a file that satisfies + conditions is removed or added during the call of this function, whether + a path name for that file be included is unspecified. .. index:: single: **; in glob-style wildcards If *recursive* is true, the pattern "``**``" will match any files and zero or - more directories and subdirectories. If the pattern is followed by an - ``os.sep``, only directories and subdirectories match. + more directories, subdirectories and symbolic links to directories. If the + pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not + match. + + .. audit-event:: glob.glob pathname,recursive glob.glob .. note:: Using the "``**``" pattern in large directory trees may consume @@ -65,6 +70,8 @@ For example, ``'[?]'`` matches the character ``'?'``. Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. + .. audit-event:: glob.glob pathname,recursive glob.iglob + .. function:: escape(pathname) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index a16c7cd4d7cd6e..f5da6ecac2dcf3 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -109,10 +109,10 @@ More condensed: Using :func:`new` with an algorithm provided by OpenSSL: - >>> h = hashlib.new('ripemd160') + >>> h = hashlib.new('sha512_256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() - 'cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc' + '19197dc4d03829df858011c6c87600f994a858103bbc19005f20987aa19a97e2' Hashlib provides the following constant attributes: diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index dc994b07c35c00..57ac8bb16120f5 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -14,12 +14,13 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. -.. function:: new(key, msg=None, digestmod=None) +.. function:: new(key, msg=None, digestmod='') Return a new hmac object. *key* is a bytes or bytearray object giving the secret key. If *msg* is present, the method call ``update(msg)`` is made. *digestmod* is the digest name, digest constructor or module for the HMAC - object to use. It supports any name suitable to :func:`hashlib.new`. + object to use. It may be any name suitable to :func:`hashlib.new`. + Despite its argument position, it is required. .. versionchanged:: 3.4 Parameter *key* can be a bytes or bytearray object. @@ -28,6 +29,8 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. .. deprecated-removed:: 3.4 3.8 MD5 as implicit default digest for *digestmod* is deprecated. + The digestmod parameter is now required. Pass it as a keyword + argument to avoid awkwardness when you do not have an initial msg. .. function:: digest(key, msg, digest) @@ -127,7 +130,6 @@ This module also provides the following helper function: a timing attack could theoretically reveal information about the types and lengths of *a* and *b*—but not their values. - .. versionadded:: 3.3 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index beaa720d732b4c..be31c3c07154f4 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -20,7 +20,7 @@ HTTPS protocols. It is normally not used directly --- the module .. seealso:: - The `Requests package `_ + The `Requests package `_ is recommended for a higher-level HTTP client interface. .. note:: @@ -94,6 +94,11 @@ The module provides the following classes: :func:`ssl._create_unverified_context` can be passed to the *context* parameter. + .. versionchanged:: 3.8 + This class now enables TLS 1.3 + :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or + when *cert_file* is passed with a custom *context*. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. @@ -511,8 +516,8 @@ Here is an example session that uses the ``GET`` method:: >>> # The following example demonstrates reading data in chunks. >>> conn.request("GET", "/") >>> r1 = conn.getresponse() - >>> while not r1.closed: - ... print(r1.read(200)) # 200 bytes + >>> while chunk := r1.read(200): + ... print(repr(chunk)) b'\n", + print("Hi. The current entry content is:", self.contents.get()) + root = tk.Tk() + myapp = App(root) + myapp.mainloop() The Window Manager ^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 5ba31feae1444d..6967d7509657b3 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -871,8 +871,8 @@ widget commands. | | remaining values are assumed empty. If there are more values | | | than columns, the extra values are ignored. | +--------+---------------------------------------------------------------+ - | open | True/False value indicating whether the item's children should| - | | be displayed or hidden. | + | open | ``True``/``False`` value indicating whether the item's | + | | children should be displayed or hidden. | +--------+---------------------------------------------------------------+ | tags | A list of tags associated with this item. | +--------+---------------------------------------------------------------+ @@ -997,7 +997,7 @@ ttk.Treeview The minimum width of the column in pixels. The treeview widget will not make the column any smaller than specified by this option when the widget is resized or the user drags a column. - * stretch: True/False + * stretch: ``True``/``False`` Specifies whether the column's width should be adjusted when the widget is resized. * width: width diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 5c641ef46d145e..7f598cd38d7f8c 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -29,17 +29,17 @@ functions. The functions mirror definitions in the Python C header files. .. function:: ISTERMINAL(x) - Return true for terminal token values. + Return ``True`` for terminal token values. .. function:: ISNONTERMINAL(x) - Return true for non-terminal token values. + Return ``True`` for non-terminal token values. .. function:: ISEOF(x) - Return true if *x* is the marker indicating the end of input. + Return ``True`` if *x* is the marker indicating the end of input. The token constants are: @@ -70,6 +70,7 @@ the :mod:`tokenize` module. .. data:: TYPE_COMMENT + :noindex: Token value indicating that a type comment was recognized. Such tokens are only produced when :func:`ast.parse()` is invoked with @@ -87,7 +88,7 @@ the :mod:`tokenize` module. now tokenized as :data:`NAME` tokens. .. versionchanged:: 3.8 - Added :data:`TYPE_COMMENT`. + Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`. Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed to support parsing older Python versions for :func:`ast.parse` with ``feature_version`` set to 6 or lower). diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index b208ba46d17d99..11f569df2e7cde 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -13,7 +13,7 @@ The :mod:`tokenize` module provides a lexical scanner for Python source code, implemented in Python. The scanner in this module returns comments as tokens -as well, making it useful for implementing "pretty-printers," including +as well, making it useful for implementing "pretty-printers", including colorizers for on-screen displays. To simplify token stream handling, all :ref:`operator ` and @@ -278,3 +278,22 @@ The exact token type names can be displayed using the :option:`-e` option: 4,10-4,11: RPAR ')' 4,11-4,12: NEWLINE '\n' 5,0-5,0: ENDMARKER '' + +Example of tokenizing a file programmatically, reading unicode +strings instead of bytes with :func:`generate_tokens`:: + + import tokenize + + with tokenize.open('hello.py') as f: + tokens = tokenize.generate_tokens(f.readline) + for token in tokens: + print(token) + +Or reading bytes directly with :func:`.tokenize`:: + + import tokenize + + with open('hello.py', 'rb') as f: + tokens = tokenize.tokenize(f.readline) + for token in tokens: + print(token) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index 85fec6830006c7..09bc1441487e03 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -153,47 +153,47 @@ Programmatic Interface count information. *timing* enables a timestamp relative to when tracing was started to be displayed. - .. method:: run(cmd) + .. method:: run(cmd) - Execute the command and gather statistics from the execution with - the current tracing parameters. *cmd* must be a string or code object, - suitable for passing into :func:`exec`. + Execute the command and gather statistics from the execution with + the current tracing parameters. *cmd* must be a string or code object, + suitable for passing into :func:`exec`. - .. method:: runctx(cmd, globals=None, locals=None) + .. method:: runctx(cmd, globals=None, locals=None) - Execute the command and gather statistics from the execution with the - current tracing parameters, in the defined global and local - environments. If not defined, *globals* and *locals* default to empty - dictionaries. + Execute the command and gather statistics from the execution with the + current tracing parameters, in the defined global and local + environments. If not defined, *globals* and *locals* default to empty + dictionaries. - .. method:: runfunc(func, *args, **kwds) + .. method:: runfunc(func, *args, **kwds) - Call *func* with the given arguments under control of the :class:`Trace` - object with the current tracing parameters. + Call *func* with the given arguments under control of the :class:`Trace` + object with the current tracing parameters. - .. method:: results() + .. method:: results() - Return a :class:`CoverageResults` object that contains the cumulative - results of all previous calls to ``run``, ``runctx`` and ``runfunc`` - for the given :class:`Trace` instance. Does not reset the accumulated - trace results. + Return a :class:`CoverageResults` object that contains the cumulative + results of all previous calls to ``run``, ``runctx`` and ``runfunc`` + for the given :class:`Trace` instance. Does not reset the accumulated + trace results. .. class:: CoverageResults A container for coverage results, created by :meth:`Trace.results`. Should not be created directly by the user. - .. method:: update(other) + .. method:: update(other) - Merge in data from another :class:`CoverageResults` object. + Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None) - Write coverage results. Set *show_missing* to show lines that had no - hits. Set *summary* to include in the output the coverage summary per - module. *coverdir* specifies the directory into which the coverage - result files will be output. If ``None``, the results for each source - file are placed in its directory. + Write coverage results. Set *show_missing* to show lines that had no + hits. Set *summary* to include in the output the coverage summary per + module. *coverdir* specifies the directory into which the coverage + result files will be output. If ``None``, the results for each source + file are placed in its directory. A simple example demonstrating the use of the programmatic interface:: diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 2d327c02540995..000c0ee9405901 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -202,10 +202,8 @@ ignoring ```` and ```` files:: print("Top %s lines" % limit) for index, stat in enumerate(top_stats[:limit], 1): frame = stat.traceback[0] - # replace "/path/to/module/file.py" with "module/file.py" - filename = os.sep.join(frame.filename.split(os.sep)[-2:]) print("#%s: %s:%s: %.1f KiB" - % (index, filename, frame.lineno, stat.size / 1024)) + % (index, frame.filename, frame.lineno, stat.size / 1024)) line = linecache.getline(frame.filename, frame.lineno).strip() if line: print(' %s' % line) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 7f9f0c34386799..6a9d61916ad1a5 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -662,7 +662,7 @@ Tell Turtle's state Return the angle between the line from turtle position to position specified by (x,y), the vector or the other turtle. This depends on the turtle's start - orientation which depends on the mode - "standard"/"world" or "logo"). + orientation which depends on the mode - "standard"/"world" or "logo". .. doctest:: :skipif: _tkinter is None @@ -913,8 +913,8 @@ Color control Set pencolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the outline of that polygon is drawn with the - newly set pencolor. + If turtleshape is a polygon, the outline of that polygon is drawn with the + newly set pencolor. .. doctest:: :skipif: _tkinter is None @@ -962,8 +962,8 @@ Color control Set fillcolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the interior of that polygon is drawn - with the newly set fillcolor. + If turtleshape is a polygon, the interior of that polygon is drawn + with the newly set fillcolor. .. doctest:: :skipif: _tkinter is None @@ -1001,8 +1001,8 @@ Color control Equivalent to ``pencolor(colorstring1)`` and ``fillcolor(colorstring2)`` and analogously if the other input format is used. - If turtleshape is a polygon, outline and interior of that polygon is drawn - with the newly set colors. + If turtleshape is a polygon, outline and interior of that polygon is drawn + with the newly set colors. .. doctest:: :skipif: _tkinter is None @@ -1051,6 +1051,11 @@ Filling Fill the shape drawn after the last call to :func:`begin_fill`. + Whether or not overlap regions for self-intersecting polygons + or multiple shapes are filled depends on the operating system graphics, + type of overlap, and number of overlaps. For example, the Turtle star + above may be either all yellow or have some white regions. + .. doctest:: :skipif: _tkinter is None @@ -1064,6 +1069,7 @@ More drawing control ~~~~~~~~~~~~~~~~~~~~ .. function:: reset() + :noindex: Delete the turtle's drawings from the screen, re-center the turtle and set variables to the default values. @@ -1085,6 +1091,7 @@ More drawing control .. function:: clear() + :noindex: Delete the turtle's drawings from the screen. Do not move turtle. State and position of the turtle as well as drawings of other turtles are not affected. @@ -1098,7 +1105,7 @@ More drawing control :param font: a triple (fontname, fontsize, fonttype) Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or right") and with the given + position according to *align* ("left", "center" or "right") and with the given font. If *move* is true, the pen is moved to the bottom-right corner of the text. By default, *move* is ``False``. @@ -1185,7 +1192,7 @@ Appearance :func:`shapesize`. - "noresize": no adaption of the turtle's appearance takes place. - resizemode("user") is called by :func:`shapesize` when used with arguments. + ``resizemode("user")`` is called by :func:`shapesize` when used with arguments. .. doctest:: :skipif: _tkinter is None @@ -1323,7 +1330,7 @@ Appearance matrix as a tuple of 4 elements. Otherwise set the given elements and transform the turtleshape according to the matrix consisting of first row t11, t12 and - second row t21, 22. The determinant t11 * t22 - t12 * t21 must not be + second row t21, t22. The determinant t11 * t22 - t12 * t21 must not be zero, otherwise an error is raised. Modify stretchfactor, shearfactor and tiltangle according to the given matrix. @@ -1357,6 +1364,7 @@ Using events ------------ .. function:: onclick(fun, btn=1, add=None) + :noindex: :param fun: a function with two arguments which will be called with the coordinates of the clicked point on the canvas @@ -1505,7 +1513,7 @@ Special Turtle methods :param size: an integer or ``None`` - Set or disable undobuffer. If *size* is an integer an empty undobuffer of + Set or disable undobuffer. If *size* is an integer, an empty undobuffer of given size is installed. *size* gives the maximum number of turtle actions that can be undone by the :func:`undo` method/function. If *size* is ``None``, the undobuffer is disabled. @@ -1813,7 +1821,7 @@ Using screen events existing bindings are removed. Example for a TurtleScreen instance named ``screen`` and a Turtle instance - named turtle: + named ``turtle``: .. doctest:: :skipif: _tkinter is None @@ -2040,7 +2048,7 @@ Methods specific to Screen, not inherited from TurtleScreen .. function:: exitonclick() - Bind bye() method to mouse clicks on the Screen. + Bind ``bye()`` method to mouse clicks on the Screen. If the value "using_IDLE" in the configuration dictionary is ``False`` diff --git a/Doc/library/types.rst b/Doc/library/types.rst index a21fb44dda5dec..ef19031910fd74 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -34,7 +34,7 @@ Dynamic Type Creation freshly created class namespace. It should accept the class namespace as its sole argument and update the namespace directly with the class contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: ns``. + in ``lambda ns: None``. .. versionadded:: 3.3 @@ -109,6 +109,11 @@ Standard names are defined for the following types: The type of user-defined functions and functions created by :keyword:`lambda` expressions. + .. audit-event:: function.__new__ code types.FunctionType + + The audit event only occurs for direct instantiation of function objects, + and is not raised for normal compilation. + .. data:: GeneratorType @@ -132,12 +137,23 @@ Standard names are defined for the following types: .. versionadded:: 3.6 -.. data:: CodeType +.. class:: CodeType(**kwargs) .. index:: builtin: compile The type for code objects such as returned by :func:`compile`. + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType + + Note that the audited arguments may not match the names or positions + required by the initializer. The audit event only occurs for direct + instantiation of code objects, and is not raised for normal compilation. + + .. method:: CodeType.replace(**kwargs) + + Return a copy of the code object with new values for the specified fields. + + .. versionadded:: 3.8 .. data:: CellType @@ -193,7 +209,7 @@ Standard names are defined for the following types: .. class:: ModuleType(name, doc=None) - The type of :term:`modules `. Constructor takes the name of the + The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. .. note:: @@ -208,12 +224,23 @@ Standard names are defined for the following types: The :term:`loader` which loaded the module. Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__loader__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. .. attribute:: __name__ - The name of the module. + The name of the module. Expected to match + :attr:`importlib.machinery.ModuleSpec.name`. .. attribute:: __package__ @@ -222,9 +249,26 @@ Standard names are defined for the following types: to ``''``, else it should be set to the name of the package (which can be :attr:`__name__` if the module is a package itself). Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__package__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. + .. attribute:: __spec__ + + A record of the the module's import-system-related state. Expected to be + an instance of :class:`importlib.machinery.ModuleSpec`. + + .. versionadded:: 3.4 + .. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) @@ -339,7 +383,9 @@ Additional Utility Classes and Functions return "{}({})".format(type(self).__name__, ", ".join(items)) def __eq__(self, other): - return self.__dict__ == other.__dict__ + if isinstance(self, SimpleNamespace) and isinstance(other, SimpleNamespace): + return self.__dict__ == other.__dict__ + return NotImplemented ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. However, for a structured record type use :func:`~collections.namedtuple` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1a766c29a57a53..9d80a209060c87 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -10,10 +10,9 @@ .. note:: - The typing module has been included in the standard library on a - :term:`provisional basis `. New features might - be added and API may change even between minor releases if deemed - necessary by the core developers. + The Python runtime does not enforce function and variable type annotations. + They can be used by third party tools such as type checkers, IDEs, linters, + etc. -------------- @@ -232,8 +231,8 @@ A user-defined class can be defined as a generic class. single type parameter ``T`` . This also makes ``T`` valid as a type within the class body. -The :class:`Generic` base class uses a metaclass that defines -:meth:`__getitem__` so that ``LoggedVar[t]`` is valid as a type:: +The :class:`Generic` base class defines :meth:`__class_getitem__` so that +``LoggedVar[t]`` is valid as a type:: from typing import Iterable @@ -308,9 +307,10 @@ User defined generic type aliases are also supported. Examples:: def inproduct(v: Vec[T]) -> T: # Same as Iterable[Tuple[T, T]] return sum(x*y for x, y in v) -The metaclass used by :class:`Generic` is a subclass of :class:`abc.ABCMeta`. -A generic class can be an ABC by including abstract methods or properties, -and generic classes can also have ABCs as base classes without a metaclass +.. versionchanged:: 3.7 + :class:`Generic` no longer has a custom metaclass. + +A user-defined generic class can have ABCs as base classes without a metaclass conflict. Generic metaclasses are not supported. The outcome of parameterizing generics is cached, and most types in the typing module are hashable and comparable for equality. @@ -324,7 +324,7 @@ every type as being compatible with :data:`Any` and :data:`Any` as being compatible with every type. This means that it is possible to perform any operation or method call on a -value of type on :data:`Any` and assign it to any variable:: +value of type :data:`Any` and assign it to any variable:: from typing import Any @@ -402,10 +402,10 @@ Initially :pep:`484` defined Python static type system as using a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. This requirement previously also applied to abstract base classes, such as -:class:`Iterable`. The problem with this approach is that a class had +:class:`~collections.abc.Iterable`. The problem with this approach is that a class had to be explicitly marked to support them, which is unpythonic and unlike what one would normally do in idiomatic dynamically typed Python code. -For example, this conforms to the :pep:`484`:: +For example, this conforms to :pep:`484`:: from typing import Sized, Iterable, Iterator @@ -570,7 +570,7 @@ The module defines the following classes, functions and decorators: :ref:`type variables `, and unions of any of these types. For example:: - def new_non_team_user(user_class: Type[Union[BaseUser, ProUser]]): ... + def new_non_team_user(user_class: Type[Union[BasicUser, ProUser]]): ... ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. @@ -672,7 +672,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ByteString`. This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + and :class:`memoryview` of byte sequences. As a shorthand for this type, :class:`bytes` can be used to annotate arguments of any of the types mentioned above. @@ -733,7 +733,7 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.5.2 -.. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) +.. class:: Coroutine(Awaitable[V_co], Generic[T_co, T_contra, V_co]) A generic version of :class:`collections.abc.Coroutine`. The variance and order of type variables @@ -959,7 +959,7 @@ The module defines the following classes, functions and decorators: .. versionchanged:: 3.6.1 Added support for default values, methods, and docstrings. - .. versionchanged:: 3.8 + .. deprecated-removed:: 3.8 3.9 Deprecated the ``_field_types`` attribute in favor of the more standard ``__annotations__`` attribute which has the same information. @@ -996,14 +996,35 @@ The module defines the following classes, functions and decorators: Point2D = TypedDict('Point2D', x=int, y=int, label=str) Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) - See :pep:`589` for more examples and detailed rules of using ``TypedDict`` - with type checkers. + By default, all keys must be present in a TypedDict. It is possible + to override this by specifying totality. + Usage:: + + class point2D(TypedDict, total=False): + x: int + y: int + + This means that a point2D TypedDict can have any of the keys omitted. A type + checker is only expected to support a literal False or True as the value of + the total argument. True is the default, and makes all items defined in the + class body be required. + + See :pep:`589` for more examples and detailed rules of using ``TypedDict``. .. versionadded:: 3.8 -.. function:: NewType(typ) +.. class:: ForwardRef + + A class used for internal typing representation of string forward references. + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + a user, but may be used by introspection tools. + + .. versionadded:: 3.7.4 + +.. function:: NewType(name, tp) - A helper function to indicate a distinct types to a typechecker, + A helper function to indicate a distinct type to a typechecker, see :ref:`distinct`. At runtime it returns a function that returns its argument. Usage:: @@ -1034,8 +1055,8 @@ The module defines the following classes, functions and decorators: a dictionary constructed by merging all the ``__annotations__`` along ``C.__mro__`` in reverse order. -.. function:: get_origin(typ) -.. function:: get_args(typ) +.. function:: get_origin(tp) +.. function:: get_args(tp) Provide basic introspection for generic types and special typing forms. @@ -1148,7 +1169,7 @@ The module defines the following classes, functions and decorators: Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. This raises :exc:`TypeError` when applied to a non-protocol class. This allows a simple-minded structural check, very similar to "one trick ponies" - in :mod:`collections.abc` such as :class:`Iterable`. For example:: + in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: @runtime_checkable class Closable(Protocol): @@ -1238,7 +1259,8 @@ The module defines the following classes, functions and decorators: .. data:: Tuple Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. + with the first item of type X and the second of type Y. The type of + the empty tuple can be written as ``Tuple[()]``. Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index ee790c0cd00bbc..225384cf391e99 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -22,7 +22,7 @@ this database is compiled from the `UCD version 12.1.0 The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" -`_. It defines the +`_. It defines the following functions: diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 811f0fb1ce9397..24a18c68484686 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -12,8 +12,9 @@ .. testsetup:: + import asyncio import unittest - from unittest.mock import Mock, MagicMock, patch, call, sentinel + from unittest.mock import Mock, MagicMock, AsyncMock, patch, call, sentinel class SomeClass: attribute = 'this is a doctest' @@ -276,6 +277,47 @@ function returns is what the call returns: 2 +Mocking asynchronous iterators +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock +:ref:`async-iterators` through ``__aiter__``. The :attr:`~Mock.return_value` +attribute of ``__aiter__`` can be used to set the return values to be used for +iteration. + + >>> mock = MagicMock() # AsyncMock also works here + >>> mock.__aiter__.return_value = [1, 2, 3] + >>> async def main(): + ... return [i async for i in mock] + ... + >>> asyncio.run(main()) + [1, 2, 3] + + +Mocking asynchronous context manager +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock +:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. +By default, ``__aenter__`` and ``__aexit__`` are ``AsyncMock`` instances that +return an async function. + + >>> class AsyncContextManager: + ... async def __aenter__(self): + ... return self + ... async def __aexit__(self, exc_type, exc, tb): + ... pass + ... + >>> mock_instance = MagicMock(AsyncContextManager()) # AsyncMock also works here + >>> async def main(): + ... async with mock_instance as result: + ... pass + ... + >>> asyncio.run(main()) + >>> mock_instance.__aenter__.assert_awaited_once() + >>> mock_instance.__aexit__.assert_awaited_once() + + Creating a Mock from an Existing Object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -851,7 +893,7 @@ Here's an example implementation: ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) - ... return super(CopyingMock, self).__call__(*args, **kwargs) + ... return super().__call__(*args, **kwargs) ... >>> c = CopyingMock(return_value=None) >>> arg = set() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 46e8ef38ab11bd..34966b38a0dcd8 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -286,7 +286,7 @@ the *new_callable* argument to :func:`patch`. used to set attributes on the mock after it is created. See the :meth:`configure_mock` method for details. - .. method:: assert_called(*args, **kwargs) + .. method:: assert_called() Assert that the mock was called at least once. @@ -297,7 +297,7 @@ the *new_callable* argument to :func:`patch`. .. versionadded:: 3.6 - .. method:: assert_called_once(*args, **kwargs) + .. method:: assert_called_once() Assert that the mock was called exactly once. @@ -317,8 +317,8 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_with(*args, **kwargs) - This method is a convenient way of asserting that calls are made in a - particular way: + This method is a convenient way of asserting that the last call has been + made in a particular way: >>> mock = Mock() >>> mock.method(1, 2, 3, test='wow') @@ -360,7 +360,7 @@ the *new_callable* argument to :func:`patch`. assert the mock has been called with the specified calls. The :attr:`mock_calls` list is checked for the calls. - If *any_order* is false (the default) then the calls must be + If *any_order* is false then the calls must be sequential. There can be extra calls before or after the specified calls. @@ -514,7 +514,6 @@ the *new_callable* argument to :func:`patch`. >>> mock.call_count 2 - .. attribute:: return_value Set this to configure the value returned by calling the mock: @@ -648,6 +647,9 @@ the *new_callable* argument to :func:`patch`. arguments and make more complex assertions. See :ref:`calls as tuples `. + .. versionchanged:: 3.8 + Added ``args`` and ``kwargs`` properties. + .. attribute:: call_args_list @@ -866,7 +868,7 @@ object:: True The result of ``mock()`` is an async function which will have the outcome - of ``side_effect`` or ``return_value``: + of ``side_effect`` or ``return_value`` after it has been awaited: - if ``side_effect`` is a function, the async function will return the result of that function, @@ -874,7 +876,7 @@ object:: exception, - if ``side_effect`` is an iterable, the async function will return the next value of the iterable, however, if the sequence of result is - exhausted, ``StopIteration`` is raised immediately, + exhausted, ``StopAsyncIteration`` is raised immediately, - if ``side_effect`` is not defined, the async function will return the value defined by ``return_value``, hence, by default, the async function returns a new :class:`AsyncMock` object. @@ -891,21 +893,51 @@ object:: >>> mock() # doctest: +SKIP + + Setting the *spec* of a :class:`Mock`, :class:`MagicMock`, or :class:`AsyncMock` + to a class with asynchronous and synchronous functions will automatically + detect the synchronous functions and set them as :class:`MagicMock` (if the + parent mock is :class:`AsyncMock` or :class:`MagicMock`) or :class:`Mock` (if + the parent mock is :class:`Mock`). All asynchronous functions will be + :class:`AsyncMock`. + + >>> class ExampleClass: + ... def sync_foo(): + ... pass + ... async def async_foo(): + ... pass + ... + >>> a_mock = AsyncMock(ExampleClass) + >>> a_mock.sync_foo + + >>> a_mock.async_foo + + >>> mock = Mock(ExampleClass) + >>> mock.sync_foo + + >>> mock.async_foo + + + .. versionadded:: 3.8 + .. method:: assert_awaited() - Assert that the mock was awaited at least once. + Assert that the mock was awaited at least once. Note that this is separate + from the object having been called, the ``await`` keyword must be used: >>> mock = AsyncMock() - >>> async def main(): - ... await mock() + >>> async def main(coroutine_mock): + ... await coroutine_mock ... - >>> asyncio.run(main()) + >>> coroutine_mock = mock() + >>> mock.called + True >>> mock.assert_awaited() - >>> mock_2 = AsyncMock() - >>> mock_2.assert_awaited() Traceback (most recent call last): ... AssertionError: Expected mock to have been awaited. + >>> asyncio.run(main(coroutine_mock)) + >>> mock.assert_awaited() .. method:: assert_awaited_once() @@ -978,11 +1010,11 @@ object:: Assert the mock has been awaited with the specified calls. The :attr:`await_args_list` list is checked for the awaits. - If *any_order* is False (the default) then the awaits must be + If *any_order* is false then the awaits must be sequential. There can be extra calls before or after the specified awaits. - If *any_order* is True then the awaits can be in any order, but + If *any_order* is true then the awaits can be in any order, but they must all appear in :attr:`await_args_list`. >>> mock = AsyncMock() @@ -990,14 +1022,15 @@ object:: ... await mock(*args, **kwargs) ... >>> calls = [call("foo"), call("bar")] - >>> mock.assert_has_calls(calls) + >>> mock.assert_has_awaits(calls) Traceback (most recent call last): ... - AssertionError: Calls not found. + AssertionError: Awaits not found. Expected: [call('foo'), call('bar')] + Actual: [] >>> asyncio.run(main('foo')) >>> asyncio.run(main('bar')) - >>> mock.assert_has_calls(calls) + >>> mock.assert_has_awaits(calls) .. method:: assert_not_awaited() @@ -1307,8 +1340,10 @@ patch is patched with a *new* object. When the function/with statement exits the patch is undone. - If *new* is omitted, then the target is replaced with a - :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is + If *new* is omitted, then the target is replaced with an + :class:`AsyncMock` if the patched object is an async function or + a :class:`MagicMock` otherwise. + If :func:`patch` is used as a decorator and *new* is omitted, the created mock is passed in as an extra argument to the decorated function. If :func:`patch` is used as a context manager the created mock is returned by the context manager. @@ -1326,8 +1361,8 @@ patch patch to pass in the object being mocked as the spec/spec_set object. *new_callable* allows you to specify a different class, or callable object, - that will be called to create the *new* object. By default :class:`MagicMock` is - used. + that will be called to create the *new* object. By default :class:`AsyncMock` + is used for async functions and :class:`MagicMock` for the rest. A more powerful form of *spec* is *autospec*. If you set ``autospec=True`` then the mock will be created with a spec from the object being replaced. @@ -1491,6 +1526,10 @@ work as expected:: ... >>> test() +.. versionchanged:: 3.8 + + :func:`patch` now returns an :class:`AsyncMock` if the target is an async function. + patch.object ~~~~~~~~~~~~ @@ -1552,15 +1591,36 @@ patch.dict :func:`patch.dict` can also be called with arbitrary keyword arguments to set values in the dictionary. - :func:`patch.dict` can be used as a context manager, decorator or class - decorator. When used as a class decorator :func:`patch.dict` honours - ``patch.TEST_PREFIX`` for choosing which methods to wrap. - .. versionchanged:: 3.8 :func:`patch.dict` now returns the patched dictionary when used as a context manager. +:func:`patch.dict` can be used as a context manager, decorator or class +decorator: + + >>> foo = {} + >>> @patch.dict(foo, {'newkey': 'newvalue'}) + ... def test(): + ... assert foo == {'newkey': 'newvalue'} + >>> test() + >>> assert foo == {} + +When used as a class decorator :func:`patch.dict` honours +``patch.TEST_PREFIX`` (default to ``'test'``) for choosing which methods to wrap: + + >>> import os + >>> import unittest + >>> from unittest.mock import patch + >>> @patch.dict('os.environ', {'newkey': 'newvalue'}) + ... class TestSample(unittest.TestCase): + ... def test_sample(self): + ... self.assertEqual(os.environ['newkey'], 'newvalue') + +If you want to use a different prefix for your test, you can inform the +patchers of the different prefix by setting ``patch.TEST_PREFIX``. For +more details about how to change the value of see :ref:`test-prefix`. + :func:`patch.dict` can be used to add members to a dictionary, or simply let a test change a dictionary, and ensure the dictionary is restored when the test ends. @@ -1773,6 +1833,8 @@ builtin :func:`ord`:: 101 +.. _test-prefix: + TEST_PREFIX ~~~~~~~~~~~ @@ -2027,20 +2089,20 @@ to change the default. Methods and their defaults: -* ``__lt__``: NotImplemented -* ``__gt__``: NotImplemented -* ``__le__``: NotImplemented -* ``__ge__``: NotImplemented -* ``__int__``: 1 -* ``__contains__``: False -* ``__len__``: 0 -* ``__iter__``: iter([]) -* ``__exit__``: False -* ``__aexit__``: False -* ``__complex__``: 1j -* ``__float__``: 1.0 -* ``__bool__``: True -* ``__index__``: 1 +* ``__lt__``: ``NotImplemented`` +* ``__gt__``: ``NotImplemented`` +* ``__le__``: ``NotImplemented`` +* ``__ge__``: ``NotImplemented`` +* ``__int__``: ``1`` +* ``__contains__``: ``False`` +* ``__len__``: ``0`` +* ``__iter__``: ``iter([])`` +* ``__exit__``: ``False`` +* ``__aexit__``: ``False`` +* ``__complex__``: ``1j`` +* ``__float__``: ``1.0`` +* ``__bool__``: ``True`` +* ``__index__``: ``1`` * ``__hash__``: default hash for the mock * ``__str__``: default str for the mock * ``__sizeof__``: default sizeof for the mock @@ -2275,6 +2337,12 @@ See :ref:`auto-speccing` for examples of how to use auto-speccing with :func:`create_autospec` and the *autospec* argument to :func:`patch`. +.. versionchanged:: 3.8 + + :func:`create_autospec` now returns an :class:`AsyncMock` if the target is + an async function. + + ANY ~~~ diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 5ec4b40856ae91..cdac92709161ee 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -56,8 +56,8 @@ test runner Kent Beck's original paper on testing frameworks using the pattern shared by :mod:`unittest`. - `Nose `_ and `pytest `_ - Third-party unittest frameworks with a lighter-weight syntax for writing + `pytest `_ + Third-party unittest framework with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. `The Python Testing Tools Taxonomy `_ @@ -73,7 +73,7 @@ test runner for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as `Buildbot `_, `Jenkins `_ - or `Hudson `_. + or `Travis-CI `_, or `AppVeyor `_. .. _unittest-minimal-example: @@ -330,7 +330,9 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. .. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages `. + Test discovery supports :term:`namespace packages ` + for start directory. Note that you need to the top level directory too. + (e.g. ``python -m unittest discover -s root/namespace -t root``). .. _organizing-tests: @@ -593,8 +595,9 @@ The following decorators and exception implement test skipping and expected fail .. decorator:: expectedFailure - Mark the test as an expected failure. If the test fails it will be - considered a success. If the test passes, it will be considered a failure. + Mark the test as an expected failure or error. If the test fails or errors + it will be considered a success. If the test passes, it will be considered + a failure. .. exception:: SkipTest(reason) @@ -896,8 +899,7 @@ Test cases .. method:: assertIs(first, second, msg=None) assertIsNot(first, second, msg=None) - Test that *first* and *second* evaluate (or don't evaluate) to the - same object. + Test that *first* and *second* are (or are not) the same object. .. versionadded:: 3.1 @@ -910,10 +912,10 @@ Test cases .. versionadded:: 3.1 - .. method:: assertIn(first, second, msg=None) - assertNotIn(first, second, msg=None) + .. method:: assertIn(member, container, msg=None) + assertNotIn(member, container, msg=None) - Test that *first* is (or is not) in *second*. + Test that *member* is (or is not) in *container*. .. versionadded:: 3.1 @@ -1088,7 +1090,8 @@ Test cases If given, *logger* should be a :class:`logging.Logger` object or a :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages. + logger, which will catch all messages that were not blocked by a + non-propagating descendent logger. If given, *level* should be either a numeric logging level or its string equivalent (for example either ``"ERROR"`` or @@ -1476,18 +1479,94 @@ Test cases after :meth:`setUpClass` if :meth:`setUpClass` raises an exception. It is responsible for calling all the cleanup functions added by - :meth:`addCleanupClass`. If you need cleanup functions to be called + :meth:`addClassCleanup`. If you need cleanup functions to be called *prior* to :meth:`tearDownClass` then you can call - :meth:`doCleanupsClass` yourself. + :meth:`doClassCleanups` yourself. - :meth:`doCleanupsClass` pops methods off the stack of cleanup + :meth:`doClassCleanups` pops methods off the stack of cleanup functions one at a time, so it can be called at any time. .. versionadded:: 3.8 +.. class:: IsolatedAsyncioTestCase(methodName='runTest') + This class provides an API similar to :class:`TestCase` and also accepts + coroutines as test functions. + .. versionadded:: 3.8 + + .. coroutinemethod:: asyncSetUp() + + Method called to prepare the test fixture. This is called after :meth:`setUp`. + This is called immediately before calling the test method; other than + :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method + will be considered an error rather than a test failure. The default implementation + does nothing. + + .. coroutinemethod:: asyncTearDown() + + Method called immediately after the test method has been called and the + result recorded. This is called before :meth:`tearDown`. This is called even if + the test method raised an exception, so the implementation in subclasses may need + to be particularly careful about checking internal state. Any exception, other than + :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be + considered an additional error rather than a test failure (thus increasing + the total number of reported errors). This method will only be called if + the :meth:`asyncSetUp` succeeds, regardless of the outcome of the test method. + The default implementation does nothing. + + .. method:: addAsyncCleanup(function, /, *args, **kwargs) + + This method accepts a coroutine that can be used as a cleanup function. + + .. method:: run(result=None) + + Sets up a new event loop to run the test, collecting the result into + the :class:`TestResult` object passed as *result*. If *result* is + omitted or ``None``, a temporary result object is created (by calling + the :meth:`defaultTestResult` method) and used. The result object is + returned to :meth:`run`'s caller. At the end of the test all the tasks + in the event loop are cancelled. + + + An example illustrating the order:: + + from unittest import IsolatedAsyncioTestCase + + events = [] + + + class Test(IsolatedAsyncioTestCase): + + + def setUp(self): + events.append("setUp") + + async def asyncSetUp(self): + self._async_connection = await AsyncConnection() + events.append("asyncSetUp") + + async def test_response(self): + events.append("test_response") + response = await self._async_connection.get("https://example.com") + self.assertEqual(response.status_code, 200) + self.addAsyncCleanup(self.on_cleanup) + + def tearDown(self): + events.append("tearDown") + + async def asyncTearDown(self): + await self._async_connection.close() + events.append("asyncTearDown") + + async def on_cleanup(self): + events.append("cleanup") + + if __name__ == "__main__": + unittest.main() + + After running the test, ``events`` would contain ``["setUp", "asyncSetUp", "test_response", "asyncTearDown", "tearDown", "cleanup"]``. .. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None) @@ -1772,11 +1851,15 @@ Loading and running tests .. versionchanged:: 3.4 Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - Discovery works for :term:`namespace packages `. - Paths are sorted before being imported so that execution order is - the same even if the underlying file system's ordering is not - dependent on file name. + not errors. + + .. versionchanged:: 3.4 + *start_dir* can be a :term:`namespace packages `. + + .. versionchanged:: 3.4 + Paths are sorted before being imported so that execution order is the + same even if the underlying file system's ordering is not dependent + on file name. .. versionchanged:: 3.5 Found packages are now checked for ``load_tests`` regardless of @@ -1869,7 +1952,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents an expected failure - of the test case. + or error of the test case. .. attribute:: unexpectedSuccesses @@ -1995,8 +2078,8 @@ Loading and running tests .. method:: addExpectedFailure(test, err) - Called when the test case *test* fails, but was marked with the - :func:`expectedFailure` decorator. + Called when the test case *test* fails or errors, but was marked with + the :func:`expectedFailure` decorator. The default implementation appends a tuple ``(test, formatted_err)`` to the instance's :attr:`expectedFailures` attribute, where *formatted_err* diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 49276daa7ff43f..7dda121f26164f 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -68,15 +68,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + >>> from urllib.parse import urlparse + >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + >>> urlparse('www.cwi.nl/%7Eguido/Python.html') + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', + >>> urlparse('help/Python.html') + ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be @@ -138,15 +138,19 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + >>> from urllib.parse import urlparse + >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> u + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + >>> u._replace(scheme='http') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + + .. warning:: + :func:`urlparse` does not perform validation. See :ref:`URL parsing + security ` for details. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -165,7 +169,7 @@ or on combining URL components into a URL string. now raise :exc:`ValueError`. -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a @@ -190,6 +194,9 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the + query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` parameter set to ``True``) to convert such dictionaries into query strings. @@ -201,8 +208,14 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.8.8 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.8.8 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of @@ -226,6 +239,9 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the + query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into query strings. @@ -235,6 +251,12 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.8.8 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.8.8 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + .. function:: urlunparse(parts) @@ -294,6 +316,15 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 + control and space characters are stripped from the URL. ``\n``, + ``\r`` and tab ``\t`` characters are removed from the URL at any position. + + .. warning:: + + :func:`urlsplit` does not perform validation. See :ref:`URL parsing + security ` for details. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. @@ -302,6 +333,13 @@ or on combining URL components into a URL string. Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.8.10 + ASCII newline and tab characters are stripped from the URL. + + .. versionchanged:: 3.8.17 + Leading WHATWG C0 control and space characters are stripped from the URL. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -377,6 +415,27 @@ or on combining URL components into a URL string. or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned without changes. +.. _url-parsing-security: + +URL parsing security +-------------------- + +The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of +inputs. They may not raise errors on inputs that other applications consider +invalid. They may also succeed on some inputs that might not be considered +URLs elsewhere. Their purpose is for practical functionality rather than +purity. + +Instead of raising an exception on unusual input, they may instead return some +component parts as empty strings. Or components may contain more than perhaps +they should. + +We recommend that users of these APIs where the values may be used anywhere +with security implications code defensively. Do some verification within your +code before trusting a returned component part. Does that ``scheme`` make +sense? Is that a sensible ``path``? Is there anything strange about that +``hostname``? etc. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes @@ -529,7 +588,7 @@ task isn't already covered by the URL parsing functions above. .. versionchanged:: 3.7 Moved from :rfc:`2396` to :rfc:`3986` for quoting URL strings. "~" is now - included in the set of reserved characters. + included in the set of unreserved characters. The optional *encoding* and *errors* parameters specify how to deal with non-ASCII characters, as accepted by the :meth:`str.encode` method. @@ -650,6 +709,10 @@ task isn't already covered by the URL parsing functions above. .. seealso:: + `WHATWG`_ - URL Living standard + Working Group for the URL Standard that defines URLs, domains, IP addresses, the + application/x-www-form-urlencoded format, and their API. + :rfc:`3986` - Uniform Resource Identifiers This is the current standard (STD66). Any changes to urllib.parse module should conform to this. Certain deviations could be observed, which are @@ -673,3 +736,5 @@ task isn't already covered by the URL parsing functions above. :rfc:`1738` - Uniform Resource Locators (URL) This specifies the formal syntax and semantics of absolute URLs. + +.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index a53c969ec629a5..0edf116fe754f5 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -18,7 +18,7 @@ authentication, redirections, cookies and more. .. seealso:: - The `Requests package `_ + The `Requests package `_ is recommended for a higher-level HTTP client interface. @@ -95,9 +95,9 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. - .. audit-event:: urllib.Request "fullurl data headers method" + .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen - The default opener raises an :func:`auditing event ` + The default opener raises an :ref:`auditing event ` ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, ``method`` taken from the request object. @@ -227,7 +227,7 @@ The following classes are provided: is not None, ``Content-Type: application/x-www-form-urlencoded`` will be added as a default. - The final two arguments are only of interest for correct handling + The next two arguments are only of interest for correct handling of third-party HTTP cookies: *origin_req_host* should be the request-host of the origin @@ -954,7 +954,7 @@ tracking URIs for which authentication credentials should always be sent. If *is_authenticated* is specified as ``True``, *realm* is ignored. -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) +.. method:: HTTPPasswordMgrWithPriorAuth.find_user_password(realm, authuri) Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 4f083a3181e7a9..593ba96cf38be1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -27,7 +27,7 @@ See :pep:`405` for more information about Python virtual environments. .. seealso:: `Python Packaging User Guide: Creating and using virtual environments - `__ + `__ Creating virtual environments @@ -47,7 +47,7 @@ Creating virtual environments A virtual environment is a directory tree which contains Python executable files and other files which indicate that it is a virtual environment. - Common installation tools such as ``Setuptools`` and ``pip`` work as + Common installation tools such as setuptools_ and pip_ work as expected with virtual environments. In other words, when a virtual environment is active, they install Python packages into the virtual environment without needing to be told to do so explicitly. @@ -64,24 +64,25 @@ Creating virtual environments Python installation). When a virtual environment is active, any options that change the - installation path will be ignored from all distutils configuration files to - prevent projects being inadvertently installed outside of the virtual - environment. + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. When working in a command shell, users can make a virtual environment active by running an ``activate`` script in the virtual environment's executables - directory (the precise filename is shell-dependent), which prepends the - virtual environment's directory for executables to the ``PATH`` environment - variable for the running shell. There should be no need in other - circumstances to activate a virtual environment—scripts installed into - virtual environments have a "shebang" line which points to the virtual - environment's Python interpreter. This means that the script will run with - that interpreter regardless of the value of ``PATH``. On Windows, "shebang" - line processing is supported if you have the Python Launcher for Windows - installed (this was added to Python in 3.3 - see :pep:`397` for more - details). Thus, double-clicking an installed script in a Windows Explorer - window should run the script with the correct interpreter without there - needing to be any reference to its virtual environment in ``PATH``. + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. .. _venv-api: @@ -130,20 +131,20 @@ creation according to their needs, the :class:`EnvBuilder` class. Added the ``prompt`` parameter Creators of third-party virtual environment tools will be free to use the - provided ``EnvBuilder`` class as a base class. + provided :class:`EnvBuilder` class as a base class. The returned env-builder is an object which has a method, ``create``: .. method:: create(env_dir) - This method takes as required argument the path (absolute or relative to - the current directory) of the target directory which is to contain the + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the virtual environment. The ``create`` method will either create the environment in the specified directory, or raise an appropriate exception. - The ``create`` method of the ``EnvBuilder`` class illustrates the hooks - available for subclass customization:: + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: def create(self, env_dir): """ @@ -267,9 +268,9 @@ subclass which installs setuptools and pip into a created virtual environment:: This builder installs setuptools and pip so that you can pip or easy_install other packages into the created virtual environment. - :param nodist: If True, setuptools and pip are not installed into the + :param nodist: If true, setuptools and pip are not installed into the created virtual environment. - :param nopip: If True, pip is not installed into the created + :param nopip: If true, pip is not installed into the created virtual environment. :param progress: If setuptools or pip are installed, the progress of the installation can be monitored by passing a progress @@ -385,7 +386,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + url = 'https://bootstrap.pypa.io/get-pip.py' self.install_script(context, 'pip', url) def main(args=None): @@ -471,3 +472,7 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online `_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a481a3509d4ec8..9c1743cad23cb7 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -491,7 +491,7 @@ Available Functions Available Context Managers -------------------------- -.. class:: catch_warnings(\*, record=False, module=None) +.. class:: catch_warnings(*, record=False, module=None) A context manager that copies and, upon exit, restores the warnings filter and the :func:`showwarning` function. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index a28d71060f3836..28f437e1a3a940 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -65,8 +65,8 @@ exposed by the :mod:`weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects `, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code +frozensets, some :term:`file objects `, :term:`generators `, +type objects, sockets, arrays, deques, regular expression pattern objects, and code objects. .. versionchanged:: 3.2 @@ -80,9 +80,10 @@ support weak references but can add support through subclassing:: obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). +.. impl-detail:: + + Other built-in types such as :class:`tuple` and :class:`int` do not support weak + references even when subclassed. Extension types can easily be made to support weak references; see :ref:`weakref-support`. @@ -162,13 +163,6 @@ Extension types can easily be made to support weak references; see application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. - .. note:: - - Caution: Because a :class:`WeakKeyDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakKeyDictionary` because actions - performed by the program during iteration may cause items in the - dictionary to vanish "by magic" (as a side effect of garbage collection). :class:`WeakKeyDictionary` objects have an additional method that exposes the internal references directly. The references are not guaranteed to @@ -188,13 +182,6 @@ than needed. Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists any more. - .. note:: - - Caution: Because a :class:`WeakValueDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakValueDictionary` because actions performed - by the program during iteration may cause items in the dictionary to vanish "by - magic" (as a side effect of garbage collection). :class:`WeakValueDictionary` objects have an additional method that has the same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` @@ -326,12 +313,6 @@ objects. types. -.. exception:: ReferenceError - - Exception raised when a proxy object is used but the underlying object has been - collected. This is the same as the standard :exc:`ReferenceError` exception. - - .. seealso:: :pep:`205` - Weak References @@ -397,7 +378,7 @@ the referent is accessed:: class ExtendedRef(weakref.ref): def __init__(self, ob, callback=None, /, **annotations): - super(ExtendedRef, self).__init__(ob, callback) + super().__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): setattr(self, k, v) @@ -406,7 +387,7 @@ the referent is accessed:: """Return a pair containing the referent and the number of times the reference has been called. """ - ob = super(ExtendedRef, self).__call__() + ob = super().__call__() if ob is not None: self.__counter += 1 ob = (ob, self.__counter) diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 9dc5551398ee82..b7bfb655a71579 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -64,6 +64,8 @@ The following functions are defined: may work and start the operating system's associated program. However, this is neither supported nor portable. + .. audit-event:: webbrowser.open url webbrowser.open + .. function:: open_new(url) diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index e9c02610227331..487856a3ac6c60 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -53,6 +53,8 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.ConnectRegistry computer_name,key winreg.ConnectRegistry + .. versionchanged:: 3.3 See :ref:`above `. @@ -75,6 +77,10 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKey + + .. audit-event:: winreg.OpenKey/result key winreg.CreateKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -103,6 +109,10 @@ This module offers the following functions: The return value is the handle of the opened key. If the function fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKeyEx + + .. audit-event:: winreg.OpenKey/result key winreg.CreateKeyEx + .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -124,6 +134,8 @@ This module offers the following functions: If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an :exc:`OSError` exception is raised. + .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -158,6 +170,8 @@ This module offers the following functions: On unsupported Windows versions, :exc:`NotImplementedError` is raised. + .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKeyEx + .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -173,6 +187,8 @@ This module offers the following functions: *value* is a string that identifies the value to remove. + .. audit-event:: winreg.DeleteValue key,value winreg.DeleteValue + .. function:: EnumKey(key, index) @@ -187,6 +203,8 @@ This module offers the following functions: typically called repeatedly until an :exc:`OSError` exception is raised, indicating, no more values are available. + .. audit-event:: winreg.EnumKey key,index winreg.EnumKey + .. versionchanged:: 3.3 See :ref:`above `. @@ -220,6 +238,8 @@ This module offers the following functions: | | :meth:`SetValueEx`) | +-------+--------------------------------------------+ + .. audit-event:: winreg.EnumValue key,index winreg.EnumValue + .. versionchanged:: 3.3 See :ref:`above `. @@ -235,6 +255,8 @@ This module offers the following functions: >>> ExpandEnvironmentStrings('%windir%') 'C:\\Windows' + .. audit-event:: winreg.ExpandEnvironmentStrings str winreg.ExpandEnvironmentStrings + .. function:: FlushKey(key) @@ -279,6 +301,8 @@ This module offers the following functions: If *key* is a handle returned by :func:`ConnectRegistry`, then the path specified in *file_name* is relative to the remote computer. + .. audit-event:: winreg.LoadKey key,sub_key,file_name winreg.LoadKey + .. function:: OpenKey(key, sub_key, reserved=0, access=KEY_READ) OpenKeyEx(key, sub_key, reserved=0, access=KEY_READ) @@ -300,6 +324,10 @@ This module offers the following functions: If the function fails, :exc:`OSError` is raised. + .. audit-event:: winreg.OpenKey key,sub_key,access winreg.OpenKey + + .. audit-event:: winreg.OpenKey/result key winreg.OpenKey + .. versionchanged:: 3.2 Allow the use of named arguments. @@ -330,6 +358,8 @@ This module offers the following functions: | | nanoseconds since Jan 1, 1601. | +-------+---------------------------------------------+ + .. audit-event:: winreg.QueryInfoKey key winreg.QueryInfoKey + .. function:: QueryValue(key, sub_key) @@ -343,10 +373,12 @@ This module offers the following functions: value set by the :func:`SetValue` method for the key identified by *key*. Values in the registry have name, type, and data components. This method - retrieves the data for a key's first value that has a NULL name. But the + retrieves the data for a key's first value that has a ``NULL`` name. But the underlying API call doesn't return the type, so always use :func:`QueryValueEx` if possible. + .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValue + .. function:: QueryValueEx(key, value_name) @@ -370,6 +402,8 @@ This module offers the following functions: | | :meth:`SetValueEx`) | +-------+-----------------------------------------+ + .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValueEx + .. function:: SaveKey(key, file_name) @@ -391,7 +425,9 @@ This module offers the following functions: `__ for more details. - This function passes NULL for *security_attributes* to the API. + This function passes ``NULL`` for *security_attributes* to the API. + + .. audit-event:: winreg.SaveKey key,file_name winreg.SaveKey .. function:: SetValue(key, sub_key, type, value) @@ -419,6 +455,8 @@ This module offers the following functions: The key identified by the *key* parameter must have been opened with :const:`KEY_SET_VALUE` access. + .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValue + .. function:: SetValueEx(key, value_name, reserved, type, value) @@ -447,6 +485,8 @@ This module offers the following functions: bytes) should be stored as files with the filenames stored in the configuration registry. This helps the registry perform efficiently. + .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValueEx + .. function:: DisableReflectionKey(key) @@ -456,13 +496,15 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. If the key is not on the reflection list, the function succeeds but has no effect. Disabling reflection for a key does not affect reflection of any subkeys. + .. audit-event:: winreg.DisableReflectionKey key winreg.DisableReflectionKey + .. function:: EnableReflectionKey(key) @@ -471,11 +513,13 @@ This module offers the following functions: *key* is an already open key, or one of the predefined :ref:`HKEY_* constants `. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. Restoring reflection for a key does not affect reflection of any subkeys. + .. audit-event:: winreg.EnableReflectionKey key winreg.EnableReflectionKey + .. function:: QueryReflectionKey(key) @@ -486,9 +530,11 @@ This module offers the following functions: Returns ``True`` if reflection is disabled. - Will generally raise :exc:`NotImplemented` if executed on a 32-bit + Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating system. + .. audit-event:: winreg.QueryReflectionKey key winreg.QueryReflectionKey + .. _constants: @@ -741,8 +787,11 @@ integer handle, and also disconnect the Windows handle from the handle object. handle is not closed. You would call this function when you need the underlying Win32 handle to exist beyond the lifetime of the handle object. + .. audit-event:: winreg.PyHKEY.Detach key winreg.PyHKEY.Detach + + .. method:: PyHKEY.__enter__() - PyHKEY.__exit__(\*exc_info) + PyHKEY.__exit__(*exc_info) The HKEY object implements :meth:`~object.__enter__` and :meth:`~object.__exit__` and thus supports the context protocol for the diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 6edd0714b9df3b..e92a689de0b9ba 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -144,7 +144,7 @@ also provides these miscellaneous utilities: .. function:: is_hop_by_hop(header_name) - Return true if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by + Return ``True`` if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by :rfc:`2616`. @@ -480,8 +480,8 @@ input, output, and error streams. rarely used and is not guaranteed by WSGI. On IIS<7, though, the setting can only be made on a vhost level, affecting all other script mappings, many of which break when exposed to the ``PATH_TRANSLATED`` bug. - For this reason IIS<7 is almost never deployed with the fix. (Even IIS7 - rarely uses it because there is still no UI for it.) + For this reason IIS<7 is almost never deployed with the fix (Even IIS7 + rarely uses it because there is still no UI for it.). There is no way for CGI code to tell whether the option was set, so a separate handler class is provided. It is used in the same way as diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 18519a75a54611..98454e18ff1f87 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -210,7 +210,7 @@ DOM Level 2 added the ability to create new :class:`Document` and .. method:: DOMImplementation.hasFeature(feature, version) - Return true if the feature identified by the pair of strings *feature* and + Return ``True`` if the feature identified by the pair of strings *feature* and *version* is implemented. @@ -335,17 +335,17 @@ All of the components of an XML document are subclasses of :class:`Node`. .. method:: Node.hasAttributes() - Returns true if the node has any attributes. + Return ``True`` if the node has any attributes. .. method:: Node.hasChildNodes() - Returns true if the node has any child nodes. + Return ``True`` if the node has any child nodes. .. method:: Node.isSameNode(other) - Returns true if *other* refers to the same node as this node. This is especially + Return ``True`` if *other* refers to the same node as this node. This is especially useful for DOM implementations which use any sort of proxy architecture (because more than one object can refer to the same node). @@ -604,12 +604,12 @@ of that class. .. method:: Element.hasAttribute(name) - Returns true if the element has an attribute named by *name*. + Return ``True`` if the element has an attribute named by *name*. .. method:: Element.hasAttributeNS(namespaceURI, localName) - Returns true if the element has an attribute named by *namespaceURI* and + Return ``True`` if the element has an attribute named by *namespaceURI* and *localName*. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index c4667315793e4c..e4aedd1f9f0823 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -249,12 +249,18 @@ We can remove elements using :meth:`Element.remove`. Let's say we want to remove all countries with a rank higher than 50:: >>> for country in root.findall('country'): + ... # using root.findall() to avoid removal during traversal ... rank = int(country.find('rank').text) ... if rank > 50: ... root.remove(country) ... >>> tree.write('output.xml') +Note that concurrent modification while iterating can lead to problems, +just like when iterating and modifying Python lists or dicts. +Therefore, the example first collects all matching elements with +``root.findall()``, and only then iterates over the list of matches. + Our XML now looks like this: .. code-block:: xml @@ -574,8 +580,8 @@ Functions .. function:: iselement(element) - Checks if an object appears to be a valid element object. *element* is an - element instance. Returns a true value if this is an element object. + Check if an object appears to be a valid element object. *element* is an + element instance. Return ``True`` if this is an element object. .. function:: iterparse(source, events=None, parser=None) @@ -659,7 +665,7 @@ Functions .. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -677,9 +683,13 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostring` function now preserves the attribute order + specified by the user. + .. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, + xml_declaration=None, default_namespace=None, \ short_empty_elements=True) Generates a string representation of an XML element, including all @@ -700,6 +710,10 @@ Functions .. versionadded:: 3.8 The *xml_declaration* and *default_namespace* parameters. + .. versionchanged:: 3.8 + The :func:`tostringlist` function now preserves the attribute order + specified by the user. + .. function:: XML(text, parser=None) @@ -718,6 +732,95 @@ Functions :class:`Element` instance and a dictionary. +.. _elementtree-xinclude: + +XInclude support +---------------- + +This module provides limited support for +`XInclude directives `_, via the :mod:`xml.etree.ElementInclude` helper module. This module can be used to insert subtrees and text strings into element trees, based on information in the tree. + +Example +^^^^^^^ + +Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include. + +.. code-block:: xml + + + + + + +By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax. + +To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module: + +.. code-block:: python + + from xml.etree import ElementTree, ElementInclude + + tree = ElementTree.parse("document.xml") + root = tree.getroot() + + ElementInclude.include(root) + +The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this: + +.. code-block:: xml + + + This is a paragraph. + + +If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required. + +To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text": + +.. code-block:: xml + + + + Copyright (c) . + + +The result might look something like: + +.. code-block:: xml + + + Copyright (c) 2003. + + +Reference +--------- + +.. _elementinclude-functions: + +Functions +^^^^^^^^^ + +.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) + + Default loader. This default loader reads an included resource from disk. *href* is a URL. + *parse* is for parse mode either "xml" or "text". *encoding* + is an optional text encoding. If not given, encoding is ``utf-8``. Returns the + expanded resource. If the parse mode is ``"xml"``, this is an ElementTree + instance. If the parse mode is "text", this is a Unicode string. If the + loader fails, it can return None or raise an exception. + + +.. function:: xml.etree.ElementInclude.include( elem, loader=None) + + This function expands XInclude directives. *elem* is the root element. *loader* is + an optional resource loader. If omitted, it defaults to :func:`default_loader`. + If given, it should be a callable that implements the same interface as + :func:`default_loader`. Returns the expanded resource. If the parse mode is + ``"xml"``, this is an ElementTree instance. If the parse mode is "text", + this is a Unicode string. If the loader fails, it can return None or + raise an exception. + + .. _elementtree-element-objects: Element Objects @@ -930,6 +1033,36 @@ Element Objects if element is None: print("element not found") + Prior to Python 3.8, the serialisation order of the XML attributes of + elements was artificially made predictable by sorting the attributes by + their name. Based on the now guaranteed ordering of dicts, this arbitrary + reordering was removed in Python 3.8 to preserve the order in which + attributes were originally parsed or created by user code. + + In general, user code should try not to depend on a specific ordering of + attributes, given that the `XML Information Set + `_ explicitly excludes the attribute + order from conveying information. Code should be prepared to deal with + any ordering on input. In cases where deterministic XML output is required, + e.g. for cryptographic signing or test data sets, canonical serialisation + is available with the :func:`canonicalize` function. + + In cases where canonical output is not applicable but a specific attribute + order is still desirable on output, code should aim for creating the + attributes directly in the desired order, to avoid perceptual mismatches + for readers of the code. In cases where this is difficult to achieve, a + recipe like the following can be applied prior to serialisation to enforce + an order independently from the Element creation:: + + def reorder_attributes(root): + for el in root.iter(): + attrib = el.attrib + if len(attrib) > 1: + # adjust attribute order, e.g. by sorting + attribs = sorted(attrib.items()) + attrib.clear() + attrib.update(attribs) + .. _elementtree-elementtree-objects: diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index fb86b6f5564d76..e3b35162961147 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -20,7 +20,7 @@ Python's interfaces for processing XML are grouped in the ``xml`` package. The XML modules are not secure against erroneous or maliciously constructed data. If you need to parse untrusted or unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defused-packages` sections. + :ref:`defusedxml-package` sections. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -60,22 +60,26 @@ circumvent firewalls. The following table gives an overview of the known attacks and whether the various modules are vulnerable to them. -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a +========================= ================== ================== ================== ================== ================== +kind sax etree minidom pulldom xmlrpc +========================= ================== ================== ================== ================== ================== +billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) +`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe +decompression bomb Safe Safe Safe Safe **Vulnerable** +========================= ================== ================== ================== ================== ================== + +1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and + "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :data:`pyexpat.EXPAT_VERSION`. +2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns +3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.7.1, external general entities are no longer processed by +4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +5. Since Python 3.7.1, external general entities are no longer processed by default. @@ -113,9 +117,9 @@ decompression bomb The documentation for `defusedxml`_ on PyPI has further information about all known attack vectors with examples and references. -.. _defused-packages: +.. _defusedxml-package: -The :mod:`defusedxml` and :mod:`defusedexpat` Packages +The :mod:`defusedxml` Package ------------------------------------------------------ `defusedxml`_ is a pure Python package with modified subclasses of all stdlib @@ -124,16 +128,8 @@ package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -`defusedexpat`_ provides a modified libexpat and a patched -:mod:`pyexpat` module that have countermeasures against entity expansion -DoS attacks. The :mod:`defusedexpat` module still allows a sane and configurable amount of entity -expansions. The modifications may be included in some future release of Python, -but will not be included in any bugfix releases of -Python because they break backward compatibility. - .. _defusedxml: https://pypi.org/project/defusedxml/ -.. _defusedexpat: https://pypi.org/project/defusedexpat/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 728315251e0824..fb40a2b3e964e4 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -198,7 +198,7 @@ Pack up a directory into an archive, and run it. The same can be done using the :func:`create_archive` function:: >>> import zipapp - >>> zipapp.create_archive('myapp.pyz', 'myapp') + >>> zipapp.create_archive('myapp', 'myapp.pyz') To make the application directly executable on POSIX, specify an interpreter to use. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 9db9697105d6b5..97da6cab806e39 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -494,7 +494,13 @@ Path objects are traversable using the ``/`` operator. Invoke :meth:`ZipFile.open` on the current path. Accepts the same arguments as :meth:`ZipFile.open`. -.. method:: Path.listdir() + .. caution:: + + The signature on this function changes in an incompatible way + in Python 3.9. For a future-compatible version, consider using + the third-party zipp.Path package (3.0 or later). + +.. method:: Path.iterdir() Enumerate the children of the current directory. @@ -816,5 +822,45 @@ Command-line options Test whether the zipfile is valid or not. +Decompression pitfalls +---------------------- + +The extraction in zipfile module might fail due to some pitfalls listed below. + +From file itself +~~~~~~~~~~~~~~~~ + +Decompression may fail due to incorrect password / CRC checksum / ZIP format or +unsupported compression method / decryption. + +File System limitations +~~~~~~~~~~~~~~~~~~~~~~~ + +Exceeding limitations on different file systems can cause decompression failed. +Such as allowable characters in the directory entries, length of the file name, +length of the pathname, size of a single file, and number of files, etc. + +Resources limitations +~~~~~~~~~~~~~~~~~~~~~ + +The lack of memory or disk volume would lead to decompression +failed. For example, decompression bombs (aka `ZIP bomb`_) +apply to zipfile library that can cause disk volume exhaustion. + +Interruption +~~~~~~~~~~~~ + +Interruption during the decompression, such as pressing control-C or killing the +decompression process may result in incomplete decompression of the archive. + +Default behaviors of extraction +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Not knowing the default extraction behaviors +can cause unexpected decompression results. +For example, when extracting the same archive twice, +it overwrites files without asking. + +.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 2138697ff06415..8ac3fb16bdb90d 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -2,7 +2,7 @@ ===================================================== .. module:: zipimport - :synopsis: support for importing Python modules from ZIP archives. + :synopsis: Support for importing Python modules from ZIP archives. .. moduleauthor:: Just van Rossum diff --git a/Doc/license.rst b/Doc/license.rst index d877f45677549b..0de692d161066d 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -22,7 +22,7 @@ Virginia where he released several versions of the software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation; see -http://www.zope.com/). In 2001, the Python Software Foundation (PSF, see +https://www.zope.org/). In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF. @@ -72,6 +72,19 @@ make these releases possible. Terms and conditions for accessing or otherwise using Python ============================================================ +Python software and documentation are licensed under the +:ref:`PSF License Agreement `. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Agreement +and the :ref:`Zero-Clause BSD license `. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. +See :ref:`OtherLicenses` for an incomplete list of these licenses. + + +.. _PSF-license: PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------ @@ -87,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2019 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2023 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -258,6 +271,27 @@ CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 SOFTWARE. +.. _BSD0: + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON |release| DOCUMENTATION +---------------------------------------------------------------------- + +.. parsed-literal:: + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. + + +.. _OtherLicenses: + Licenses and Acknowledgements for Incorporated Software ======================================================= diff --git a/Doc/make.bat b/Doc/make.bat index e6604956ea916b..ac66b56e9a3ef7 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -13,7 +13,7 @@ if not defined SPHINXBUILD ( %PYTHON% -c "import sphinx" > nul 2> nul if errorlevel 1 ( echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install sphinx + %PYTHON% -m pip install -r requirements.txt if errorlevel 1 exit /B ) set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" @@ -30,6 +30,7 @@ if not defined BLURB ( %PYTHON% -c "import blurb" > nul 2> nul if errorlevel 1 ( echo Installing blurb with %PYTHON% + rem Should have been installed with Sphinx earlier %PYTHON% -m pip install blurb if errorlevel 1 exit /B ) @@ -54,9 +55,9 @@ if not exist "%HTMLHELP%" ( ) :skiphhcsearch -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v +if not defined DISTVERSION for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v -if "%BUILDDIR%" EQU "" set BUILDDIR=build +if not defined BUILDDIR set BUILDDIR=build rem Targets that don't require sphinx-build if "%1" EQU "" goto help @@ -117,13 +118,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 +if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" if exist ..\Misc\NEWS ( echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul ) else if exist ..\Misc\NEWS.D ( if defined BLURB ( echo.Merging Misc/NEWS with %BLURB% - if not exist build mkdir build %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" ) else ( echo.No Misc/NEWS file and Blurb is not available. @@ -131,7 +132,7 @@ if exist ..\Misc\NEWS ( ) ) -if NOT "%PAPER%" == "" ( +if defined PAPER ( set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% ) if "%1" EQU "htmlhelp" ( diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 988eec6d254e10..9dd471a0c83ec5 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -44,7 +44,8 @@ executed:: Summarizing: -.. productionlist:: + +.. productionlist:: python-grammar compound_stmt: `if_stmt` : | `while_stmt` : | `for_stmt` @@ -89,9 +90,9 @@ The :keyword:`!if` statement The :keyword:`if` statement is used for conditional execution: -.. productionlist:: - if_stmt: "if" `expression` ":" `suite` - : ("elif" `expression` ":" `suite`)* +.. productionlist:: python-grammar + if_stmt: "if" `assignment_expression` ":" `suite` + : ("elif" `assignment_expression` ":" `suite`)* : ["else" ":" `suite`] It selects exactly one of the suites by evaluating the expressions one by one @@ -115,8 +116,8 @@ The :keyword:`!while` statement The :keyword:`while` statement is used for repeated execution as long as an expression is true: -.. productionlist:: - while_stmt: "while" `expression` ":" `suite` +.. productionlist:: python-grammar + while_stmt: "while" `assignment_expression` ":" `suite` : ["else" ":" `suite`] This repeatedly tests the expression and, if it is true, executes the first @@ -151,7 +152,7 @@ The :keyword:`!for` statement The :keyword:`for` statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: -.. productionlist:: +.. productionlist:: python-grammar for_stmt: "for" `target_list` "in" `expression_list` ":" `suite` : ["else" ":" `suite`] @@ -234,7 +235,7 @@ The :keyword:`!try` statement The :keyword:`try` statement specifies exception handlers and/or cleanup code for a group of statements: -.. productionlist:: +.. productionlist:: python-grammar try_stmt: `try1_stmt` | `try2_stmt` try1_stmt: "try" ":" `suite` : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ @@ -253,7 +254,8 @@ present, must be last; it matches any exception. For an except clause with an expression, that expression is evaluated, and the clause matches the exception if the resulting object is "compatible" with the exception. An object is compatible with an exception if it is the class or a base class of the exception -object or a tuple containing an item compatible with the exception. +object, or a tuple containing an item that is the class or a base class of +the exception object. If no except clause matches the exception, the search for an exception handler continues in the surrounding code and on the invocation stack. [#]_ @@ -390,7 +392,7 @@ methods defined by a context manager (see section :ref:`context-managers`). This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` usage patterns to be encapsulated for convenient reuse. -.. productionlist:: +.. productionlist:: python-grammar with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` with_item: `expression` ["as" `target`] @@ -399,6 +401,8 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo #. The context expression (the expression given in the :token:`with_item`) is evaluated to obtain a context manager. +#. The context manager's :meth:`__enter__` is loaded for later use. + #. The context manager's :meth:`__exit__` is loaded for later use. #. The context manager's :meth:`__enter__` method is invoked. @@ -430,17 +434,41 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo value from :meth:`__exit__` is ignored, and execution proceeds at the normal location for the kind of exit that was taken. +The following code:: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to:: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: with A() as a, B() as b: - suite + SUITE -is equivalent to :: +is semantically equivalent to:: with A() as a: with B() as b: - suite + SUITE .. versionchanged:: 3.1 Support for multiple context expressions. @@ -477,7 +505,7 @@ Function definitions A function definition defines a user-defined function object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` decorators: `decorator`+ @@ -554,19 +582,25 @@ e.g.:: return penguin .. index:: + single: / (slash); function definition single: * (asterisk); function definition single: **; function definition Function call semantics are described in more detail in section :ref:`calls`. A function call always assigns values to all parameters mentioned in the parameter -list, either from position arguments, from keyword arguments, or from default +list, either from positional arguments, from keyword arguments, or from default values. If the form "``*identifier``" is present, it is initialized to a tuple receiving any excess positional parameters, defaulting to the empty tuple. If the form "``**identifier``" is present, it is initialized to a new ordered mapping receiving any excess keyword arguments, defaulting to a new empty mapping of the same type. Parameters after "``*``" or "``*identifier``" are keyword-only parameters and may only be passed -used keyword arguments. +by keyword arguments. Parameters before "``/``" are positional-only parameters +and may only be passed by positional arguments. + +.. versionchanged:: 3.8 + The ``/`` function parameter syntax may be used to indicate positional-only + parameters. See :pep:`570` for details. .. index:: pair: function; annotations @@ -639,7 +673,7 @@ Class definitions A class definition defines a class object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` inheritance: "(" [`argument_list`] ")" classname: `identifier` @@ -726,7 +760,7 @@ Coroutines Coroutine function definition ----------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` @@ -759,7 +793,7 @@ An example of a coroutine function:: The :keyword:`!async for` statement ----------------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_for_stmt: "async" `for_stmt` An :term:`asynchronous iterable` is able to call asynchronous code in its @@ -772,24 +806,25 @@ iterators. The following code:: async for TARGET in ITER: - BLOCK + SUITE else: - BLOCK2 + SUITE2 Is semantically equivalent to:: iter = (ITER) iter = type(iter).__aiter__(iter) running = True + while running: try: TARGET = await type(iter).__anext__(iter) except StopAsyncIteration: running = False else: - BLOCK + SUITE else: - BLOCK2 + SUITE2 See also :meth:`__aiter__` and :meth:`__anext__` for details. @@ -803,7 +838,7 @@ body of a coroutine function. The :keyword:`!async with` statement ------------------------------------ -.. productionlist:: +.. productionlist:: python-grammar async_with_stmt: "async" `with_stmt` An :term:`asynchronous context manager` is a :term:`context manager` that is @@ -811,23 +846,27 @@ able to suspend execution in its *enter* and *exit* methods. The following code:: - async with EXPR as VAR: - BLOCK + async with EXPRESSION as TARGET: + SUITE -Is semantically equivalent to:: +is semantically equivalent to:: - mgr = (EXPR) - aexit = type(mgr).__aexit__ - aenter = type(mgr).__aenter__(mgr) + manager = (EXPRESSION) + aexit = type(manager).__aexit__ + aenter = type(manager).__aenter__ + value = await aenter(manager) + hit_except = False - VAR = await aenter try: - BLOCK + TARGET = value + SUITE except: - if not await aexit(mgr, *sys.exc_info()): + hit_except = True + if not await aexit(manager, *sys.exc_info()): raise - else: - await aexit(mgr, None, None, None) + finally: + if not hit_except: + await aexit(manager, None, None, None) See also :meth:`__aenter__` and :meth:`__aexit__` for details. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index fa47bf1c1619b5..92058aa4ed8867 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -17,7 +17,7 @@ Objects, values and types :dfn:`Objects` are Python's abstraction for data. All data in a Python program is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer," code is also +conformance to Von Neumann's model of a "stored program computer", code is also represented by objects.) .. index:: @@ -182,6 +182,24 @@ Ellipsis related to mathematical numbers, but subject to the limitations of numerical representation in computers. + The string representations of the numeric classes, computed by + :meth:`__repr__` and :meth:`__str__`, have the following + properties: + + * They are valid numeric literals which, when passed to their + class constructor, produce an object having the value of the + original numeric. + + * The representation is in base 10, when possible. + + * Leading zeros, possibly excepting a single zero before a + decimal point, are not shown. + + * Trailing zeros, possibly excepting a single zero after a + decimal point, are not shown. + + * A sign is shown only when the number is negative. + Python distinguishes between integers, floating point numbers, and complex numbers: @@ -194,7 +212,6 @@ Ellipsis There are two types of integers: Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) memory only. For the purpose of shift and mask operations, a binary representation is assumed, and negative numbers are represented in a variant of @@ -420,6 +437,11 @@ Mappings equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index the same dictionary entry. + Dictionaries preserve insertion order, meaning that keys will be produced + in the same order they were added sequentially over the dictionary. + Replacing an existing key does not change the order, however removing a key + and re-inserting it will add it to the end instead of keeping its old place. + Dictionaries are mutable; they can be created by the ``{...}`` notation (see section :ref:`dict`). @@ -431,6 +453,11 @@ Mappings additional examples of mapping types, as does the :mod:`collections` module. + .. versionchanged:: 3.7 + Dictionaries did not preserve insertion order in versions of Python before 3.6. + In CPython 3.6, insertion order was preserved, but it was considered + an implementation detail at that time rather than a language guarantee. + Callable types .. index:: object: callable @@ -925,8 +952,8 @@ Internal types the first line number of the function; :attr:`co_lnotab` is a string encoding the mapping from bytecode offsets to line numbers (for details see the source code of the interpreter); :attr:`co_stacksize` is the - required stack size (including local variables); :attr:`co_flags` is an - integer encoding a number of flags for the interpreter. + required stack size; :attr:`co_flags` is an integer encoding a number + of flags for the interpreter. .. index:: object: generator @@ -973,6 +1000,9 @@ Internal types :attr:`f_lasti` gives the precise instruction (this is an index into the bytecode string of the code object). + Accessing ``f_code`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) @@ -1057,6 +1087,9 @@ Internal types :keyword:`try` statement with no matching except clause or with a finally clause. + Accessing ``tb_frame`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. + .. index:: single: tb_next (traceback attribute) @@ -1166,10 +1199,10 @@ Basic customization with appropriate arguments and then modifying the newly-created instance as necessary before returning it. - If :meth:`__new__` returns an instance of *cls*, then the new instance's - :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where - *self* is the new instance and the remaining arguments are the same as were - passed to :meth:`__new__`. + If :meth:`__new__` is invoked during object construction and it returns an + instance of *cls*, then the new instance’s :meth:`__init__` method + will be invoked like ``__init__(self[, ...])``, where *self* is the new instance + and the remaining arguments are the same as were passed to the object constructor. If :meth:`__new__` does not return an instance of *cls*, then the new instance's :meth:`__init__` method will not be invoked. @@ -1362,12 +1395,14 @@ Basic customization context (e.g., in the condition of an ``if`` statement), Python will call :func:`bool` on the value to determine if the result is true or false. - By default, :meth:`__ne__` delegates to :meth:`__eq__` and - inverts the result unless it is ``NotImplemented``. There are no other - implied relationships among the comparison operators, for example, - the truth of ``(x` ``object.__getattr__`` with arguments + ``obj`` and ``name``. + .. method:: object.__setattr__(self, name, value) @@ -1536,12 +1577,24 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. call the base class method with the same name, for example, ``object.__setattr__(self, name, value)``. + .. audit-event:: object.__setattr__ obj,name,value object.__setattr__ + + For certain sensitive attribute assignments, raises an + :ref:`auditing event ` ``object.__setattr__`` with arguments + ``obj``, ``name``, ``value``. + .. method:: object.__delattr__(self, name) Like :meth:`__setattr__` but for attribute deletion instead of assignment. This should only be implemented if ``del obj.name`` is meaningful for the object. + .. audit-event:: object.__delattr__ obj,name object.__delattr__ + + For certain sensitive attribute deletions, raises an + :ref:`auditing event ` ``object.__delattr__`` with arguments + ``obj`` and ``name``. + .. method:: object.__dir__(self) @@ -1566,7 +1619,7 @@ not found on a module object through the normal lookup, i.e. the module ``__dict__`` before raising an :exc:`AttributeError`. If found, it is called with the attribute name and the result is returned. -The ``__dir__`` function should accept no arguments, and return a list of +The ``__dir__`` function should accept no arguments, and return a sequence of strings that represents the names accessible on module. If present, this function overrides the standard :func:`dir` search on a module. @@ -1618,21 +1671,32 @@ refers to the attribute whose name is the key of the property in the owner class' :attr:`~object.__dict__`. -.. method:: object.__get__(self, instance, owner) +.. method:: object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute access) or + of an instance of that class (instance attribute access). The optional + *owner* argument is the owner class, while *instance* is the instance that + the attribute was accessed through, or ``None`` when the attribute is + accessed through the *owner*. - Called to get the attribute of the owner class (class attribute access) or of an - instance of that class (instance attribute access). *owner* is always the owner - class, while *instance* is the instance that the attribute was accessed through, - or ``None`` when the attribute is accessed through the *owner*. This method - should return the (computed) attribute value or raise an :exc:`AttributeError` - exception. + This method should return the computed attribute value or raise an + :exc:`AttributeError` exception. + :PEP:`252` specifies that :meth:`__get__` is callable with one or two + arguments. Python's own built-in descriptors support this specification; + however, it is likely that some third-party tools have descriptors + that require both arguments. Python's own :meth:`__getattribute__` + implementation always passes in both arguments whether they are required + or not. .. method:: object.__set__(self, instance, value) Called to set the attribute on an instance *instance* of the owner class to a new value, *value*. + Note, adding :meth:`__set__` or :meth:`__delete__` changes the kind of + descriptor to a "data descriptor". See :ref:`descriptor-invocation` for + more details. .. method:: object.__delete__(self, instance) @@ -1644,8 +1708,22 @@ class' :attr:`~object.__dict__`. Called at the time the owning class *owner* is created. The descriptor has been assigned to *name*. - .. versionadded:: 3.6 + .. note:: + + :meth:`__set_name__` is only called implicitly as part of the + :class:`type` constructor, so it will need to be called explicitly with + the appropriate parameters when a descriptor is added to a class after + initial creation:: + class A: + pass + descr = custom_descriptor() + A.attr = descr + descr.__set_name__(A, 'attr') + + See :ref:`class-object-creation` for more details. + + .. versionadded:: 3.6 The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module as specifying the class where this object was defined (setting this @@ -1696,7 +1774,7 @@ Super Binding immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. -For instance bindings, the precedence of descriptor invocation depends on the +For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`. If it does not define :meth:`__get__`, then accessing the attribute will return the descriptor @@ -1786,6 +1864,10 @@ Notes on using *__slots__* (the other bases must have empty slot layouts) - violations raise :exc:`TypeError`. +* If an iterator is used for *__slots__* then a descriptor is created for each + of the iterator's values. However, the *__slots__* attribute will be an empty + iterator. + .. _class-customization: Customizing class creation @@ -1916,7 +1998,10 @@ Preparing the class namespace Once the appropriate metaclass has been identified, then the class namespace is prepared. If the metaclass has a ``__prepare__`` attribute, it is called as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the -additional keyword arguments, if any, come from the class definition). +additional keyword arguments, if any, come from the class definition). The +``__prepare__`` method should be implemented as a :func:`classmethod`. The +namespace returned by ``__prepare__`` is passed in to ``__new__``, but when +the final class object is created the namespace is copied into a new ``dict``. If the metaclass has no ``__prepare__`` attribute, then the class namespace is initialised as an empty ordered mapping. @@ -2083,7 +2168,7 @@ Emulating callable objects .. index:: pair: call; instance Called when the instance is "called" as a function; if this method is defined, - ``x(arg1, arg2, ...)`` is a shorthand for ``x.__call__(arg1, arg2, ...)``. + ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. .. _sequence-types: @@ -2117,8 +2202,8 @@ operators. It is recommended that both mappings and sequences implement the mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and sequences implement the :meth:`__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should be the same as -:meth:`keys`; for sequences, it should iterate through the values. +through the container; for mappings, :meth:`__iter__` should iterate +through the object's keys; for sequences, it should iterate through the values. .. method:: object.__len__(self) @@ -2144,7 +2229,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as Called to implement :func:`operator.length_hint`. Should return an estimated length for the object (which may be greater or less than the actual length). - The length must be an integer ``>=`` 0. This method is purely an + The length must be an integer ``>=`` 0. The return value may also be + :const:`NotImplemented`, which is treated the same as if the + ``__length_hint__`` method didn't exist at all. This method is purely an optimization and is never required for correctness. .. versionadded:: 3.4 @@ -2230,9 +2317,9 @@ through the container; for mappings, :meth:`__iter__` should be the same as The membership test operators (:keyword:`in` and :keyword:`not in`) are normally -implemented as an iteration through a sequence. However, container objects can +implemented as an iteration through a container. However, container objects can supply the following special method with a more efficient implementation, which -also does not require the object be a sequence. +also does not require the object be iterable. .. method:: object.__contains__(self, item) @@ -2300,7 +2387,7 @@ left undefined. object.__rfloordiv__(self, other) object.__rmod__(self, other) object.__rdivmod__(self, other) - object.__rpow__(self, other) + object.__rpow__(self, other[, modulo]) object.__rlshift__(self, other) object.__rrshift__(self, other) object.__rand__(self, other) @@ -2327,10 +2414,11 @@ left undefined. .. note:: - If the right operand's type is a subclass of the left operand's type and that - subclass provides the reflected method for the operation, this method will be - called before the left operand's non-reflected method. This behavior allows - subclasses to override their ancestors' operations. + If the right operand's type is a subclass of the left operand's type and + that subclass provides a different implementation of the reflected method + for the operation, this method will be called before the left operand's + non-reflected method. This behavior allows subclasses to override their + ancestors' operations. .. method:: object.__iadd__(self, other) @@ -2360,6 +2448,13 @@ left undefined. :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact part of the data model. + .. note:: + + Due to a bug in the dispatching mechanism for ``**=``, a class that + defines :meth:`__ipow__` but returns ``NotImplemented`` would fail to + fall back to ``x.__pow__(y)`` and ``y.__rpow__(x)``. This bug is fixed + in Python 3.10. + .. method:: object.__neg__(self) object.__pos__(self) @@ -2412,8 +2507,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - If :meth:`__int__` is not defined then the built-in function :func:`int` - falls back to :meth:`__trunc__`. + The built-in function :func:`int` falls back to :meth:`__trunc__` if neither + :meth:`__int__` nor :meth:`__index__` is defined. .. _context-managers: @@ -2552,7 +2647,7 @@ Awaitable Objects ----------------- An :term:`awaitable` object generally implements an :meth:`__await__` method. -:term:`Coroutine` objects returned from :keyword:`async def` functions +:term:`Coroutine objects ` returned from :keyword:`async def` functions are awaitable. .. note:: @@ -2577,7 +2672,7 @@ are awaitable. Coroutine Objects ----------------- -:term:`Coroutine` objects are :term:`awaitable` objects. +:term:`Coroutine objects ` are :term:`awaitable` objects. A coroutine's execution can be controlled by calling :meth:`__await__` and iterating over the result. When the coroutine has finished executing and returns, the iterator raises :exc:`StopIteration`, and the exception's @@ -2722,6 +2817,6 @@ An example of an asynchronous context manager class:: method—that will instead have the opposite effect of explicitly *blocking* such fallback. -.. [#] For operands of the same type, it is assumed that if the non-reflected method - (such as :meth:`__add__`) fails the operation is not supported, which is why the - reflected method is not called. +.. [#] For operands of the same type, it is assumed that if the non-reflected + method -- such as :meth:`__add__` -- fails then the overall operation is not + supported, which is why the reflected method is not called. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8b7110615240ee..d5b34379e2b59f 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -13,7 +13,7 @@ This chapter explains the meaning of the elements of expressions in Python. be used to describe syntax, not lexical analysis. When (one alternative of) a syntax rule has the form -.. productionlist:: * +.. productionlist:: python-grammar name: `othername` and no semantics are given, the semantics of this form of ``name`` are the same @@ -28,7 +28,7 @@ Arithmetic conversions .. index:: pair: arithmetic; conversion When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type," this means that the operator +arguments are converted to a common type", this means that the operator implementation for built-in types works as follows: * If either argument is a complex number, the other is converted to complex; @@ -54,7 +54,7 @@ Atoms are the most basic elements of expressions. The simplest atoms are identifiers or literals. Forms enclosed in parentheses, brackets or braces are also categorized syntactically as atoms. The syntax for atoms is: -.. productionlist:: +.. productionlist:: python-grammar atom: `identifier` | `literal` | `enclosure` enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` : | `generator_expression` | `yield_atom` @@ -103,7 +103,7 @@ Literals Python supports string and bytes literals and various numeric literals: -.. productionlist:: +.. productionlist:: python-grammar literal: `stringliteral` | `bytesliteral` : | `integer` | `floatnumber` | `imagnumber` @@ -134,7 +134,7 @@ Parenthesized forms A parenthesized form is an optional expression list enclosed in parentheses: -.. productionlist:: +.. productionlist:: python-grammar parenth_form: "(" [`starred_expression`] ")" A parenthesized expression list yields whatever that expression list yields: if @@ -148,9 +148,8 @@ immutable, the same rules as for literals apply (i.e., two occurrences of the em tuple may or may not yield the same object). .. index:: - single: comma; tuple display - pair: tuple; display - single: , (comma); tuple display + single: comma + single: , (comma) Note that tuples are not formed by the parentheses, but rather by use of the comma operator. The exception is the empty tuple, for which parentheses *are* @@ -163,6 +162,8 @@ ambiguities and allow common typos to pass uncaught. Displays for lists, sets and dictionaries ----------------------------------------- +.. index:: single: comprehensions + For constructing a list, a set or a dictionary Python provides special syntax called "displays", each of them in two flavors: @@ -178,8 +179,8 @@ called "displays", each of them in two flavors: Common syntax elements for comprehensions are: -.. productionlist:: - comprehension: `expression` `comp_for` +.. productionlist:: python-grammar + comprehension: `assignment_expression` `comp_for` comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` comp_if: "if" `expression_nocond` [`comp_iter`] @@ -244,7 +245,7 @@ List displays A list display is a possibly empty series of expressions enclosed in square brackets: -.. productionlist:: +.. productionlist:: python-grammar list_display: "[" [`starred_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either @@ -261,6 +262,7 @@ Set displays .. index:: pair: set; display + pair: set; comprehensions object: set single: {} (curly brackets); set expression single: , (comma); expression list @@ -268,7 +270,7 @@ Set displays A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: -.. productionlist:: +.. productionlist:: python-grammar set_display: "{" (`starred_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by @@ -288,6 +290,7 @@ Dictionary displays .. index:: pair: dictionary; display + pair: dictionary; comprehensions key, datum, key/datum pair object: dictionary single: {} (curly brackets); dictionary expression @@ -297,7 +300,7 @@ Dictionary displays A dictionary display is a possibly empty series of key/datum pairs enclosed in curly braces: -.. productionlist:: +.. productionlist:: python-grammar dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" key_datum_list: `key_datum` ("," `key_datum`)* [","] key_datum: `expression` ":" `expression` | "**" `or_expr` @@ -337,6 +340,12 @@ all mutable objects.) Clashes between duplicate keys are not detected; the last datum (textually rightmost in the display) stored for a given key value prevails. +.. versionchanged:: 3.8 + Prior to Python 3.8, in dict comprehensions, the evaluation order of key + and value was not well-defined. In CPython, the value was evaluated before + the key. Starting with 3.8, the key is evaluated before the value, as + proposed by :pep:`572`. + .. _genexpr: @@ -350,7 +359,7 @@ Generator expressions A generator expression is a compact generator notation in parentheses: -.. productionlist:: +.. productionlist:: python-grammar generator_expression: "(" `expression` `comp_for` ")" A generator expression yields a new generator object. Its syntax is the same as @@ -404,7 +413,7 @@ Yield expressions pair: yield; expression pair: generator; function -.. productionlist:: +.. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] @@ -467,8 +476,8 @@ allowing any pending :keyword:`finally` clauses to execute. .. index:: single: from; yield from expression -When ``yield from `` is used, it treats the supplied expression as -a subiterator. All values produced by that subiterator are passed directly +When ``yield from `` is used, the supplied expression must be an +iterable. The values produced by iterating that iterable are passed directly to the caller of the current generator's methods. Any values passed in with :meth:`~generator.send` and any exceptions passed in with :meth:`~generator.throw` are passed to the underlying iterator if it has the @@ -701,7 +710,8 @@ which are used to control the execution of a generator function. because there is no yield expression that could receive the value. -.. coroutinemethod:: agen.athrow(type[, value[, traceback]]) +.. coroutinemethod:: agen.athrow(value) + agen.athrow(type[, value[, traceback]]) Returns an awaitable that raises an exception of type ``type`` at the point where the asynchronous generator was paused, and returns the next value @@ -741,7 +751,7 @@ Primaries Primaries represent the most tightly bound operations of the language. Their syntax is: -.. productionlist:: +.. productionlist:: python-grammar primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` @@ -756,7 +766,7 @@ Attribute references An attribute reference is a primary followed by a period and a name: -.. productionlist:: +.. productionlist:: python-grammar attributeref: `primary` "." `identifier` .. index:: @@ -794,7 +804,7 @@ Subscriptions A subscription selects an item of a sequence (string, tuple or list) or mapping (dictionary) object: -.. productionlist:: +.. productionlist:: python-grammar subscription: `primary` "[" `expression_list` "]" The primary must evaluate to an object that supports subscription (lists or @@ -850,7 +860,7 @@ A slicing selects a range of items in a sequence object (e.g., a string, tuple or list). Slicings may be used as expressions or as targets in assignment or :keyword:`del` statements. The syntax for a slicing: -.. productionlist:: +.. productionlist:: python-grammar slicing: `primary` "[" `slice_list` "]" slice_list: `slice_item` ("," `slice_item`)* [","] slice_item: `expression` | `proper_slice` @@ -900,13 +910,14 @@ Calls A call calls a callable object (e.g., a :term:`function`) with a possibly empty series of :term:`arguments `: -.. productionlist:: +.. productionlist:: python-grammar call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" argument_list: `positional_arguments` ["," `starred_and_keywords`] : ["," `keywords_arguments`] : | `starred_and_keywords` ["," `keywords_arguments`] : | `keywords_arguments` - positional_arguments: ["*"] `expression` ("," ["*"] `expression`)* + positional_arguments: positional_item ("," positional_item)* + positional_item: `assignment_expression` | "*" `expression` starred_and_keywords: ("*" `expression` | `keyword_item`) : ("," "*" `expression` | "," `keyword_item`)* keywords_arguments: (`keyword_item` | "**" `expression`) @@ -1082,7 +1093,7 @@ Await expression Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. Can only be used inside a :term:`coroutine function`. -.. productionlist:: +.. productionlist:: python-grammar await_expr: "await" `primary` .. versionadded:: 3.5 @@ -1100,7 +1111,7 @@ The power operator The power operator binds more tightly than unary operators on its left; it binds less tightly than unary operators on its right. The syntax is: -.. productionlist:: +.. productionlist:: python-grammar power: (`await_expr` | `primary`) ["**" `u_expr`] Thus, in an unparenthesized sequence of power and unary operators, the operators @@ -1133,7 +1144,7 @@ Unary arithmetic and bitwise operations All unary arithmetic and bitwise operations have the same priority: -.. productionlist:: +.. productionlist:: python-grammar u_expr: `power` | "-" `u_expr` | "+" `u_expr` | "~" `u_expr` .. index:: @@ -1177,7 +1188,7 @@ that some of these operations also apply to certain non-numeric types. Apart from the power operator, there are only two levels, one for multiplicative operators and one for additive operators: -.. productionlist:: +.. productionlist:: python-grammar m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | : `m_expr` "%" `u_expr` @@ -1273,7 +1284,7 @@ Shifting operations The shifting operations have lower priority than the arithmetic operations: -.. productionlist:: +.. productionlist:: python-grammar shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` These operators accept integers as arguments. They shift the first argument to @@ -1294,7 +1305,7 @@ Binary bitwise operations Each of the three bitwise operations has a different priority level: -.. productionlist:: +.. productionlist:: python-grammar and_expr: `shift_expr` | `and_expr` "&" `shift_expr` xor_expr: `and_expr` | `xor_expr` "^" `and_expr` or_expr: `xor_expr` | `or_expr` "|" `xor_expr` @@ -1343,7 +1354,7 @@ lower than that of any arithmetic, shifting or bitwise operation. Also unlike C, expressions like ``a < b < c`` have the interpretation that is conventional in mathematics: -.. productionlist:: +.. productionlist:: python-grammar comparison: `or_expr` (`comp_operator` `or_expr`)* comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" : | "is" ["not"] | ["not"] "in" @@ -1416,8 +1427,13 @@ built-in types. The not-a-number values ``float('NaN')`` and ``decimal.Decimal('NaN')`` are special. Any ordered comparison of a number to a not-a-number value is false. A counter-intuitive implication is that not-a-number values are not equal to - themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3``, ``x - == x``, ``x != x`` are all false. This behavior is compliant with IEEE 754. + themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3`` and + ``x == x`` are all false, while ``x != x`` is true. This behavior is + compliant with IEEE 754. + +* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that + comparisons for singletons should always be done with ``is`` or ``is not``, + never the equality operators. * Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be compared within and across their types. They compare lexicographically using @@ -1436,25 +1452,9 @@ built-in types. :exc:`TypeError`. Sequences compare lexicographically using comparison of corresponding - elements, whereby reflexivity of the elements is enforced. - - In enforcing reflexivity of elements, the comparison of collections assumes - that for a collection element ``x``, ``x == x`` is always true. Based on - that assumption, element identity is compared first, and element comparison - is performed only for distinct elements. This approach yields the same - result as a strict element comparison would, if the compared elements are - reflexive. For non-reflexive elements, the result is different than for - strict element comparison, and may be surprising: The non-reflexive - not-a-number values for example result in the following comparison behavior - when used in a list:: - - >>> nan = float('NaN') - >>> nan is nan - True - >>> nan == nan - False <-- the defined non-reflexive behavior of NaN - >>> [nan] == [nan] - True <-- list enforces reflexivity and tests identity first + elements. The built-in containers typically assume identical objects are + equal to themselves. That lets them bypass equality tests for identical + objects to improve performance and to maintain their internal invariants. Lexicographical comparison between built-in collections works as follows: @@ -1613,7 +1613,7 @@ Boolean operations pair: Conditional; expression pair: Boolean; operation -.. productionlist:: +.. productionlist:: python-grammar or_test: `and_test` | `or_test` "or" `and_test` and_test: `not_test` | `and_test` "and" `not_test` not_test: `comparison` | "not" `not_test` @@ -1649,6 +1649,34 @@ returns a boolean value regardless of the type of its argument (for example, ``not 'foo'`` produces ``False`` rather than ``''``.) +Assignment expressions +====================== + +.. productionlist:: python-grammar + assignment_expression: [`identifier` ":="] `expression` + +An assignment expression (sometimes also called a "named expression" or +"walrus") assigns an :token:`expression` to an :token:`identifier`, while also +returning the value of the :token:`expression`. + +One common use case is when handling matched regular expressions: + +.. code-block:: python + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + +.. code-block:: python + + while chunk := file.read(9000): + process(chunk) + +.. versionadded:: 3.8 + See :pep:`572` for more details about assignment expressions. + + .. _if_expr: Conditional expressions @@ -1660,7 +1688,7 @@ Conditional expressions single: if; conditional expression single: else; conditional expression -.. productionlist:: +.. productionlist:: python-grammar conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` expression_nocond: `or_test` | `lambda_expr_nocond` @@ -1687,7 +1715,7 @@ Lambdas pair: anonymous; function single: : (colon); lambda expression -.. productionlist:: +.. productionlist:: python-grammar lambda_expr: "lambda" [`parameter_list`] ":" `expression` lambda_expr_nocond: "lambda" [`parameter_list`] ":" `expression_nocond` @@ -1714,11 +1742,11 @@ Expression lists pair: expression; list single: , (comma); expression list -.. productionlist:: +.. productionlist:: python-grammar expression_list: `expression` ("," `expression`)* [","] starred_list: `starred_item` ("," `starred_item`)* [","] starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `expression` | "*" `or_expr` + starred_item: `assignment_expression` | "*" `or_expr` .. index:: object: tuple @@ -1791,6 +1819,8 @@ precedence and have a left-to-right chaining feature as described in the +-----------------------------------------------+-------------------------------------+ | Operator | Description | +===============================================+=====================================+ +| ``:=`` | Assignment expression | ++-----------------------------------------------+-------------------------------------+ | :keyword:`lambda` | Lambda expression | +-----------------------------------------------+-------------------------------------+ | :keyword:`if ` -- :keyword:`!else` | Conditional expression | @@ -1828,7 +1858,8 @@ precedence and have a left-to-right chaining feature as described in the | ``x[index]``, ``x[index:index]``, | Subscription, slicing, | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ -| ``(expressions...)``, | Binding or tuple display, | +| ``(expressions...)``, | Binding or parenthesized | +| | expression, | | ``[expressions...]``, | list display, | | ``{key: value...}``, | dictionary display, | | ``{expressions...}`` | set display | diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 0228bfb7e984c5..4027ffdd5e0c05 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -202,6 +202,8 @@ named module, the two module objects will *not* be the same. By contrast, reinitialise the module contents by rerunning the module's code. +.. _finders-and-loaders: + Finders and loaders ------------------- @@ -678,11 +680,11 @@ Here are the exact rules used: Cached bytecode invalidation ---------------------------- -Before Python loads cached bytecode from ``.pyc`` file, it checks whether the +Before Python loads cached bytecode from a ``.pyc`` file, it checks whether the cache is up-to-date with the source ``.py`` file. By default, Python does this by storing the source's last-modified timestamp and size in the cache file when writing it. At runtime, the import system then validates the cache file by -checking the stored metadata in the cache file against at source's +checking the stored metadata in the cache file against the source's metadata. Python also supports "hash-based" cache files, which store a hash of the source @@ -849,15 +851,14 @@ In order to support imports of modules and initialized packages and also to contribute portions to namespace packages, path entry finders must implement the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. -:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the +:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two arguments: the fully qualified name of the module being imported, and the (optional) target module. ``find_spec()`` returns a fully populated spec for the module. This spec will always have "loader" set (with one exception). To indicate to the import machinery that the spec represents a namespace -:term:`portion`. the path entry finder sets "loader" on the spec to -``None`` and "submodule_search_locations" to a list containing the -portion. +:term:`portion`, the path entry finder sets "submodule_search_locations" to +a list containing the portion. .. versionchanged:: 3.4 :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced @@ -873,18 +874,7 @@ portion. :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the fully qualified name of the module being imported. ``find_loader()`` returns a 2-tuple where the first item is the loader and the second item - is a namespace :term:`portion`. When the first item (i.e. the loader) is - ``None``, this means that while the path entry finder does not have a - loader for the named module, it knows that the path entry contributes to - a namespace portion for the named module. This will almost always be the - case where Python is asked to import a namespace package that has no - physical presence on the file system. When a path entry finder returns - ``None`` for the loader, the second item of the 2-tuple return value must - be a sequence, although it can be empty. - - If ``find_loader()`` returns a non-``None`` loader value, the portion is - ignored and the loader is returned from the path based finder, terminating - the search through the path entries. + is a namespace :term:`portion`. For backwards compatibility with other implementations of the import protocol, many path entry finders also support the same, @@ -913,7 +903,7 @@ the builtin :func:`__import__` function may be sufficient. This technique may also be employed at the module level to only alter the behaviour of import statements within that module. -To selectively prevent import of some modules from a hook early on the +To selectively prevent the import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), it is sufficient to raise :exc:`ModuleNotFoundError` directly from :meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index bb7e3906dba607..72e874ee98e466 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -93,7 +93,7 @@ Notation The descriptions of lexical analysis and syntax use a modified BNF grammar notation. This uses the following style of definition: -.. productionlist:: * +.. productionlist:: notation name: `lc_letter` (`lc_letter` | "_")* lc_letter: "a"..."z" diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index cc1b2f57a70e3b..f74ff7949ef786 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -296,7 +296,7 @@ Unicode Character Database as included in the :mod:`unicodedata` module. Identifiers are unlimited in length. Case is significant. -.. productionlist:: +.. productionlist:: python-grammar identifier: `xid_start` `xid_continue`* id_start: id_continue: @@ -325,7 +325,7 @@ of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode 4.1 can be found at -https://www.dcl.hpi.uni-potsdam.de/home/loewis/table-3131.html. +https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -376,11 +376,11 @@ characters: information on this convention. ``__*__`` - System-defined names. These names are defined by the interpreter and its - implementation (including the standard library). Current system names are - discussed in the :ref:`specialnames` section and elsewhere. More will likely - be defined in future versions of Python. *Any* use of ``__*__`` names, in - any context, that does not follow explicitly documented use, is subject to + System-defined names, informally known as "dunder" names. These names are + defined by the interpreter and its implementation (including the standard library). + Current system names are discussed in the :ref:`specialnames` section and elsewhere. + More will likely be defined in future versions of Python. *Any* use of ``__*__`` names, + in any context, that does not follow explicitly documented use, is subject to breakage without warning. ``__*`` @@ -412,7 +412,7 @@ String and Bytes literals String literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar stringliteral: [`stringprefix`](`shortstring` | `longstring`) stringprefix: "r" | "u" | "R" | "U" | "f" | "F" : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" @@ -424,7 +424,7 @@ String literals are described by the following lexical definitions: longstringchar: stringescapeseq: "\" -.. productionlist:: +.. productionlist:: python-grammar bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' @@ -594,11 +594,9 @@ escape sequences only recognized in string literals fall into the category of unrecognized escapes for bytes literals. .. versionchanged:: 3.6 - Unrecognized escape sequences produce a :exc:`DeprecationWarning`. - - .. versionchanged:: 3.8 - Unrecognized escape sequences produce a :exc:`SyntaxWarning`. In - some future version of Python they will be a :exc:`SyntaxError`. + Unrecognized escape sequences produce a :exc:`DeprecationWarning`. In + a future Python version they will be a :exc:`SyntaxWarning` and + eventually a :exc:`SyntaxError`. Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, ``r"\""`` is a valid string @@ -639,9 +637,11 @@ and formatted string literals may be concatenated with plain string literals. single: string; formatted literal single: string; interpolated literal single: f-string + single: fstring single: {} (curly brackets); in formatted string literal single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal + single: = (equals); for help in debugging using string literals .. _f-strings: Formatted string literals @@ -659,9 +659,9 @@ Escape sequences are decoded like in ordinary string literals (except when a literal is also marked as a raw string). After decoding, the grammar for the contents of the string is: -.. productionlist:: +.. productionlist:: python-grammar f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" + replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" f_expression: (`conditional_expression` | "*" `or_expr`) : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` @@ -673,10 +673,11 @@ The parts of the string outside curly braces are treated literally, except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced with the corresponding single curly brace. A single opening curly bracket ``'{'`` marks a replacement field, which starts with a -Python expression. After the expression, there may be a conversion field, -introduced by an exclamation point ``'!'``. A format specifier may also -be appended, introduced by a colon ``':'``. A replacement field ends -with a closing curly bracket ``'}'``. +Python expression. To display both the expression text and its value after +evaluation, (useful in debugging), an equal sign ``'='`` may be added after the +expression. A conversion field, introduced by an exclamation point ``'!'`` may +follow. A format specifier may also be appended, introduced by a colon ``':'``. +A replacement field ends with a closing curly bracket ``'}'``. Expressions in formatted string literals are treated like regular Python expressions surrounded by parentheses, with a few exceptions. @@ -687,6 +688,22 @@ strings), but they cannot contain comments. Each expression is evaluated in the context where the formatted string literal appears, in order from left to right. +.. versionchanged:: 3.7 + Prior to Python 3.7, an :keyword:`await` expression and comprehensions + containing an :keyword:`async for` clause were illegal in the expressions + in formatted string literals due to a problem with the implementation. + +When the equal sign ``'='`` is provided, the output will have the expression +text, the ``'='`` and the evaluated value. Spaces after the opening brace +``'{'``, within the expression and after the ``'='`` are all retained in the +output. By default, the ``'='`` causes the :func:`repr` of the expression to be +provided, unless there is a format specified. When a format is specified it +defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is +declared. + +.. versionadded:: 3.8 + The equal sign ``'='``. + If a conversion is specified, the result of evaluating the expression is converted before formatting. Conversion ``'!s'`` calls :func:`str` on the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. @@ -721,9 +738,22 @@ Some examples of formatted string literals:: >>> today = datetime(year=2017, month=1, day=27) >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' + >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging + 'today=January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier '0x400' + >>> foo = "bar" + >>> f"{ foo = }" # preserves whitespace + " foo = 'bar'" + >>> line = "The mill's closed" + >>> f"{line = }" + 'line = "The mill\'s closed"' + >>> f"{line = :20}" + "line = The mill's closed " + >>> f"{line = !r:20}" + 'line = "The mill\'s closed" ' + A consequence of sharing the same syntax as regular string literals is that characters in the replacement fields must not conflict with the @@ -790,7 +820,7 @@ Integer literals Integer literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ @@ -834,7 +864,7 @@ Floating point literals Floating point literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` pointfloat: [`digitpart`] `fraction` | `digitpart` "." exponentfloat: (`digitpart` | `pointfloat`) `exponent` @@ -864,7 +894,7 @@ Imaginary literals Imaginary literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex @@ -889,7 +919,7 @@ The following tokens are operators: + - * ** / // % @ - << >> & | ^ ~ + << >> & | ^ ~ := < > <= >= == != diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 0a043a90050c4e..8691cbd0a47860 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -11,7 +11,7 @@ A simple statement is comprised within a single logical line. Several simple statements may occur on a single line separated by semicolons. The syntax for simple statements is: -.. productionlist:: +.. productionlist:: python-grammar simple_stmt: `expression_stmt` : | `assert_stmt` : | `assignment_stmt` @@ -46,7 +46,7 @@ result; in Python, procedures return the value ``None``). Other uses of expression statements are allowed and occasionally useful. The syntax for an expression statement is: -.. productionlist:: +.. productionlist:: python-grammar expression_stmt: `starred_expression` An expression statement evaluates the expression list (which may be a single @@ -82,7 +82,7 @@ Assignment statements Assignment statements are used to (re)bind names to values and to modify attributes or items of mutable objects: -.. productionlist:: +.. productionlist:: python-grammar assignment_stmt: (`target_list` "=")+ (`starred_expression` | `yield_expression`) target_list: `target` ("," `target`)* [","] target: `identifier` @@ -280,7 +280,7 @@ Augmented assignment statements Augmented assignment is the combination, in a single statement, of a binary operation and an assignment statement: -.. productionlist:: +.. productionlist:: python-grammar augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) augtarget: `identifier` | `attributeref` | `subscription` | `slicing` augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" @@ -328,7 +328,7 @@ Annotated assignment statements :term:`Annotation ` assignment is the combination, in a single statement, of a variable or attribute annotation and an optional assignment statement: -.. productionlist:: +.. productionlist:: python-grammar annotated_assignment_stmt: `augtarget` ":" `expression` : ["=" (`starred_expression` | `yield_expression`)] @@ -385,7 +385,7 @@ The :keyword:`!assert` statement Assert statements are a convenient way to insert debugging assertions into a program: -.. productionlist:: +.. productionlist:: python-grammar assert_stmt: "assert" `expression` ["," `expression`] The simple form, ``assert expression``, is equivalent to :: @@ -425,7 +425,7 @@ The :keyword:`!pass` statement pair: null; operation pair: null; operation -.. productionlist:: +.. productionlist:: python-grammar pass_stmt: "pass" :keyword:`pass` is a null operation --- when it is executed, nothing happens. @@ -447,7 +447,7 @@ The :keyword:`!del` statement pair: deletion; target triple: deletion; target; list -.. productionlist:: +.. productionlist:: python-grammar del_stmt: "del" `target_list` Deletion is recursively defined very similar to the way assignment is defined. @@ -486,7 +486,7 @@ The :keyword:`!return` statement pair: function; definition pair: class; definition -.. productionlist:: +.. productionlist:: python-grammar return_stmt: "return" [`expression_list`] :keyword:`return` may only occur syntactically nested in a function definition, @@ -525,7 +525,7 @@ The :keyword:`!yield` statement single: function; generator exception: StopIteration -.. productionlist:: +.. productionlist:: python-grammar yield_stmt: `yield_expression` A :keyword:`yield` statement is semantically equivalent to a :ref:`yield @@ -560,7 +560,7 @@ The :keyword:`!raise` statement pair: raising; exception single: __traceback__ (exception attribute) -.. productionlist:: +.. productionlist:: python-grammar raise_stmt: "raise" [`expression` ["from" `expression`]] If no expressions are present, :keyword:`raise` re-raises the last exception @@ -591,10 +591,13 @@ instance, with its traceback set to its argument), like so:: __context__ (exception attribute) The ``from`` clause is used for exception chaining: if given, the second -*expression* must be another exception class or instance, which will then be -attached to the raised exception as the :attr:`__cause__` attribute (which is -writable). If the raised exception is not handled, both exceptions will be -printed:: +*expression* must be another exception class or instance. If the second +expression is an exception instance, it will be attached to the raised +exception as the :attr:`__cause__` attribute (which is writable). If the +expression is an exception class, the class will be instantiated and the +resulting exception instance will be attached to the raised exception as the +:attr:`__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed:: >>> try: ... print(1 / 0) @@ -663,7 +666,7 @@ The :keyword:`!break` statement statement: while pair: loop; statement -.. productionlist:: +.. productionlist:: python-grammar break_stmt: "break" :keyword:`break` may only occur syntactically nested in a :keyword:`for` or @@ -698,7 +701,7 @@ The :keyword:`!continue` statement pair: loop; statement keyword: finally -.. productionlist:: +.. productionlist:: python-grammar continue_stmt: "continue" :keyword:`continue` may only occur syntactically nested in a :keyword:`for` or @@ -725,7 +728,7 @@ The :keyword:`!import` statement exception: ImportError single: , (comma); import statement -.. productionlist:: +.. productionlist:: python-grammar import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* : | "from" `relative_module` "import" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* @@ -839,6 +842,7 @@ the :ref:`relativeimports` section. :func:`importlib.import_module` is provided to support applications that determine dynamically the modules to be loaded. +.. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import .. _future: @@ -858,7 +862,7 @@ that introduce incompatible changes to the language. It allows use of the new features on a per-module basis before the release in which the feature becomes standard. -.. productionlist:: * +.. productionlist:: python-grammar future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] : ("," `feature` ["as" `identifier`])* : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] @@ -873,8 +877,8 @@ can appear before a future statement are: * blank lines, and * other future statements. -The only feature in Python 3.7 that requires using the future statement is -``annotations``. +The only feature that requires using the future statement is +``annotations`` (see :pep:`563`). All historical features enabled by the future statement are still recognized by Python 3. The list includes ``absolute_import``, ``division``, @@ -936,7 +940,7 @@ The :keyword:`!global` statement triple: global; name; binding single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar global_stmt: "global" `identifier` ("," `identifier`)* The :keyword:`global` statement is a declaration which holds for the entire @@ -981,7 +985,7 @@ The :keyword:`!nonlocal` statement .. index:: statement: nonlocal single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* .. XXX add when implemented diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index d5ffb37b2e58cd..319c9de484241e 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -66,7 +66,7 @@ File input All input read from non-interactive files has the same form: -.. productionlist:: +.. productionlist:: python-grammar file_input: (NEWLINE | `statement`)* This syntax is used in the following situations: @@ -85,7 +85,7 @@ Interactive input Input in interactive mode is parsed using the following grammar: -.. productionlist:: +.. productionlist:: python-grammar interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE Note that a (top-level) compound statement must be followed by a blank line in @@ -103,5 +103,5 @@ Expression input :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: -.. productionlist:: +.. productionlist:: python-grammar eval_input: `expression_list` NEWLINE* diff --git a/Doc/requirements.txt b/Doc/requirements.txt new file mode 100644 index 00000000000000..aa615e6bf3141f --- /dev/null +++ b/Doc/requirements.txt @@ -0,0 +1,18 @@ +# Requirements to build the Python documentation + +# Sphinx version is pinned so that new versions that introduce new warnings +# won't suddenly cause build failures. Updating the version is fine as long +# as no warnings are raised by doing so. +sphinx==2.4.4 +# Docutils version is pinned to a version compatible with Sphinx +# version 2.4.4. It can be removed after bumping Sphinx version to at +# least 3.5.4. +docutils==0.17.1 +# Jinja version is pinned to a version compatible with Sphinx version 2.4.4. +jinja2==3.0.3 + +blurb + +# The theme used by the documentation is stored separately, so we need +# to install that as well. +python-docs-theme diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index fa8244a8fd318e..76c9d920cbe31f 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -79,9 +79,9 @@ def add_annotations(self, app, doctree): classes=['stableabi'])) if par['objtype'] != 'function': continue - if not par[0].has_key('names') or not par[0]['names']: + if not par[0].has_key('ids') or not par[0]['ids']: continue - name = par[0]['names'][0] + name = par[0]['ids'][0] if name.startswith("c."): name = name[2:] entry = self.get(name) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8bf7eb699c4e72..359d76451493d0 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -22,20 +22,29 @@ from sphinx import addnodes from sphinx.builders import Builder +try: + from sphinx.errors import NoUri +except ImportError: + from sphinx.environment import NoUri from sphinx.locale import translators -from sphinx.util import status_iterator +from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title -from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter, TextTranslator from sphinx.writers.latex import LaTeXTranslator -from sphinx.domains.python import PyModulelevel, PyClassmember + +try: + from sphinx.domains.python import PyFunction, PyMethod +except ImportError: + from sphinx.domains.python import PyClassmember as PyMethod + from sphinx.domains.python import PyModulelevel as PyFunction # Support for checking for suspicious markup import suspicious -ISSUE_URI = 'https://bugs.python.org/issue%s' +ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' +GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' SOURCE_URI = 'https://github.com/python/cpython/tree/3.8/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists @@ -45,47 +54,38 @@ Body.enum.converters['lowerroman'] = \ Body.enum.converters['upperroman'] = lambda x: None -# monkey-patch HTML and LaTeX translators to keep doctest blocks in the -# doctest docs themselves -orig_visit_literal_block = HTMLTranslator.visit_literal_block -orig_depart_literal_block = LaTeXTranslator.depart_literal_block - - -def new_visit_literal_block(self, node): - meta = self.builder.env.metadata[self.builder.current_docname] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_visit_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -def new_depart_literal_block(self, node): - meta = self.builder.env.metadata[self.curfilestack[-1]] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_depart_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -HTMLTranslator.visit_literal_block = new_visit_literal_block -LaTeXTranslator.depart_literal_block = new_depart_literal_block - # Support for marking up and linking to bugs.python.org issues def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): issue = utils.unescape(text) + # sanity check: there are no bpo issues within these two values + if 47261 < int(issue) < 400000: + msg = inliner.reporter.error(f'The BPO ID {text!r} seems too high -- ' + 'use :gh:`...` for GitHub IDs', line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] text = 'bpo-' + issue refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) return [refnode], [] +# Support for marking up and linking to GitHub issues + +def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): + issue = utils.unescape(text) + # sanity check: all GitHub issues have ID >= 32426 + # even though some of them are also valid BPO IDs + if int(issue) < 32426: + msg = inliner.reporter.error(f'The GitHub ID {text!r} seems too low -- ' + 'use :issue:`...` for BPO IDs', line=lineno) + prb = inliner.problematic(rawtext, rawtext, msg) + return [prb], [msg] + text = 'gh-' + issue + refnode = nodes.reference(text, text, refuri=GH_ISSUE_URI % issue) + return [refnode], [] + + # Support for linking to Python source files easily def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): @@ -157,7 +157,7 @@ class AuditEvent(Directive): has_content = True required_arguments = 1 - optional_arguments = 1 + optional_arguments = 2 final_argument_whitespace = True _label = [ @@ -166,21 +166,54 @@ class AuditEvent(Directive): "Raises an :ref:`auditing event ` {name} with arguments {args}.", ] + @property + def logger(self): + cls = type(self) + return logging.getLogger(cls.__module__ + "." + cls.__name__) + def run(self): + name = self.arguments[0] if len(self.arguments) >= 2 and self.arguments[1]: - args = [ - "``{}``".format(a.strip()) - for a in self.arguments[1].strip("'\"").split() - if a.strip() - ] + args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) + args = [a for a in args if a] else: args = [] label = translators['sphinx'].gettext(self._label[min(2, len(args))]) - text = label.format(name="``{}``".format(self.arguments[0]), - args=", ".join(args)) + text = label.format(name="``{}``".format(name), + args=", ".join("``{}``".format(a) for a in args if a)) - pnode = nodes.paragraph(text, classes=["audit-hook"]) + env = self.state.document.settings.env + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + + new_info = { + 'source': [], + 'args': args + } + info = env.all_audit_events.setdefault(name, new_info) + if info is not new_info: + if not self._do_args_match(info['args'], new_info['args']): + self.logger.warn( + "Mismatched arguments for audit-event {}: {!r} != {!r}" + .format(name, info['args'], new_info['args']) + ) + + ids = [] + try: + target = self.arguments[2].strip("\"'") + except (IndexError, TypeError): + target = None + if not target: + target = "audit_event_{}_{}".format( + re.sub(r'\W', '_', name), + len(info['source']), + ) + ids.append(target) + + info['source'].append((env.docname, target)) + + pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) if self.content: self.state.nested_parse(self.content, self.content_offset, pnode) else: @@ -189,6 +222,37 @@ def run(self): return [pnode] + # This list of sets are allowable synonyms for event argument names. + # If two names are in the same set, they are treated as equal for the + # purposes of warning. This won't help if number of arguments is + # different! + _SYNONYMS = [ + {"file", "path", "fd"}, + ] + + def _do_args_match(self, args1, args2): + if args1 == args2: + return True + if len(args1) != len(args2): + return False + for a1, a2 in zip(args1, args2): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in self._SYNONYMS): + continue + return False + return True + + +class audit_event_list(nodes.General, nodes.Element): + pass + + +class AuditEventListDirective(Directive): + + def run(self): + return [audit_event_list('')] + # Support for documenting decorators @@ -202,17 +266,18 @@ def needs_arglist(self): return False -class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): +class PyDecoratorFunction(PyDecoratorMixin, PyFunction): def run(self): # a decorator function is a function after all self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): +# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible +class PyDecoratorMethod(PyDecoratorMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) class PyCoroutineMixin(object): @@ -229,31 +294,31 @@ def handle_signature(self, sig, signode): return ret -class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): +class PyCoroutineFunction(PyCoroutineMixin, PyFunction): def run(self): self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): +class PyCoroutineMethod(PyCoroutineMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAwaitableFunction(PyAwaitableMixin, PyClassmember): +class PyAwaitableFunction(PyAwaitableMixin, PyFunction): def run(self): self.name = 'py:function' - return PyClassmember.run(self) + return PyFunction.run(self) -class PyAwaitableMethod(PyAwaitableMixin, PyClassmember): +class PyAwaitableMethod(PyAwaitableMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAbstractMethod(PyClassmember): +class PyAbstractMethod(PyMethod): def handle_signature(self, sig, signode): ret = super(PyAbstractMethod, self).handle_signature(sig, signode) @@ -263,7 +328,7 @@ def handle_signature(self, sig, signode): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) # Support for documenting version of removal in deprecations @@ -275,7 +340,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' def run(self): node = addnodes.versionmodified() @@ -283,7 +349,15 @@ def run(self): node['type'] = 'deprecated-removed' version = (self.arguments[0], self.arguments[1]) node['version'] = version - label = translators['sphinx'].gettext(self._label) + env = self.state.document.settings.env + current_version = tuple(int(e) for e in env.config.version.split('.')) + removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + if current_version < removed_version: + label = self._deprecated_label + else: + label = self._removed_label + + label = translators['sphinx'].gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], @@ -316,7 +390,8 @@ def run(self): # Support for including Misc/NEWS -issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') +issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)', re.I) +gh_issue_re = re.compile('(?:gh-issue-|gh-)([0-9]+)', re.I) whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") @@ -343,8 +418,9 @@ def run(self): text = 'The NEWS file is not available.' node = nodes.strong(text, text) return [node] - content = issue_re.sub(r'`bpo-\1 `__', - content) + content = issue_re.sub(r':issue:`\1`', content) + # Fallback handling for the GitHub issue + content = gh_issue_re.sub(r':gh:`\1`', content) content = whatsnew_re.sub(r'\1', content) # remove first 3 lines as they are the main heading lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] @@ -394,7 +470,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) + self.env.logger.warn('label %r not in documentation' % label) continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -458,12 +534,78 @@ def parse_pdb_command(env, sig, signode): return fullname +def process_audit_events(app, doctree, fromdocname): + for node in doctree.traverse(audit_event_list): + break + else: + return + + env = app.builder.env + + table = nodes.table(cols=3) + group = nodes.tgroup( + '', + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + cols=3, + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + row = nodes.row() + row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) + head += row + + for name in sorted(getattr(env, "all_audit_events", ())): + audit_event = env.all_audit_events[name] + + row = nodes.row() + node = nodes.paragraph('', nodes.Text(name)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, a in enumerate(audit_event['args']): + if i: + node += nodes.Text(", ") + node += nodes.literal(a, nodes.Text(a)) + row += nodes.entry('', node) + + node = nodes.paragraph() + backlinks = enumerate(sorted(set(audit_event['source'])), start=1) + for i, (doc, label) in backlinks: + if isinstance(label, str): + ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) + try: + ref['refuri'] = "{}#{}".format( + app.builder.get_relative_uri(fromdocname, doc), + label, + ) + except NoUri: + continue + node += ref + row += nodes.entry('', node) + + body += row + + for node in doctree.traverse(audit_event_list): + node.replace_self(table) + + def setup(app): app.add_role('issue', issue_role) + app.add_role('gh', gh_issue_role) app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) app.add_directive('audit-event', AuditEvent) + app.add_directive('audit-event-table', AuditEventListDirective) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) @@ -478,4 +620,5 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('doctree-resolved', process_audit_events) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 494efabc46234e..9e814fb94d2b56 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -115,10 +115,12 @@ def write_doc(self, docname, doctree): def finish(self): unused_rules = [rule for rule in self.rules if not rule.used] if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) - for rule in unused_rules: - self.logger.info(repr(rule)) + self.logger.warning( + 'Found %s/%s unused rules: %s' % ( + len(unused_rules), len(self.rules), + ''.join(repr(rule) for rule in unused_rules), + ) + ) return def check_issue(self, line, lineno, issue): @@ -151,14 +153,15 @@ def report_issue(self, text, lineno, issue): self.any_issue = True self.write_log_entry(lineno, issue, text) if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) + self.logger.warning('[%s:%d] "%s" found in "%-.120s"' % + (self.docname, lineno, issue, text)) else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( - self.docname.encode(sys.getdefaultencoding(),'replace'), - lineno, - issue.encode(sys.getdefaultencoding(),'replace'), - text.strip().encode(sys.getdefaultencoding(),'replace'))) + self.logger.warning( + '[%s:%d] "%s" found in "%-.120s"' % ( + self.docname.encode(sys.getdefaultencoding(),'replace'), + lineno, + issue.encode(sys.getdefaultencoding(),'replace'), + text.strip().encode(sys.getdefaultencoding(),'replace'))) self.app.statuscode = 1 def write_log_entry(self, lineno, issue, text): diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js deleted file mode 100644 index 346b31494e60f9..00000000000000 --- a/Doc/tools/static/switchers.js +++ /dev/null @@ -1,154 +0,0 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - var version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - var all_versions = { - '3.8': 'dev (3.8)', - '3.7': '3.7', - '3.6': '3.6', - '3.5': '3.5', - '2.7': '2.7', - }; - - var all_languages = { - 'en': 'English', - 'fr': 'French', - 'ja': 'Japanese', - 'ko': 'Korean', - 'zh-cn': 'Simplified Chinese', - }; - - function build_version_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function build_language_select(current_language) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + current_language + selected_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - url.replace('.org/' + current_language + current_version, - '.org/' + selected_version), - 'https://docs.python.org/' + current_language + selected_version, - 'https://docs.python.org/' + selected_version, - 'https://docs.python.org/' - ]); - } - } - - function on_language_switch() { - var selected_language = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - if (selected_language == 'en/') // Special 'default' case for english. - selected_language = ''; - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + selected_language + current_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://docs.python.org/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url(url) { - var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)'; - var match = url.match(language_regexp); - if (match !== null) - return match[1]; - return ''; - } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_in_url(url) { - var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)'; - var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; - var match = url.match(version_regexp); - if (match !== null) - return match[1]; - return '' - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var language_segment = language_segment_from_url(window.location.href); - var current_language = language_segment.replace(/\/+$/g, '') || 'en'; - var version = release.substr(0, 3); - var version_select = build_version_select(version, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var language_select = build_language_select(current_language); - - $('.language_switcher_placeholder').html(language_select); - $('.language_switcher_placeholder select').bind('change', on_language_switch); - }); -})(); diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 85263d47c8bba8..dd6aa38d72adcc 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -2,16 +2,16 @@ c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)" c-api/list,,:high,list[low:high] c-api/sequence,,:i2,del o[i1:i2] c-api/sequence,,:i2,o[i1:i2] +c-api/tuple,,:high,p[low:high] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,274,`,This is the description of the ``foobar`` package. +distutils/examples,267,`,This is the description of the ``foobar`` package. distutils/setupscript,,::, extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {" extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {" faq/programming,,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" -faq/programming,,::,for x in sequence[::-1]: faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," faq/windows,,:d48eceb,"Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32" @@ -198,7 +198,7 @@ library/readline,,:bind,"python:bind ^I rl_complete" library/smtplib,,:port,method must support that as well as a regular host:port library/socket,,::,'5aef:2b::8' library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" -library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) +library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" library/sqlite3,,:memory, library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" @@ -216,8 +216,6 @@ library/stdtypes,,:len,s[len(s):len(s)] library/stdtypes,,::,>>> y = m[::2] library/stdtypes,,::,>>> z = y[::-2] library/string,,`,"!""#$%&'()*+,-./:;<=>?@[\]^_`{|}~" -library/subprocess,,`,"output=`dmesg | grep hda`" -library/subprocess,,`,"output=`mycmd myarg`" library/tarfile,,:bz2, library/tarfile,,:compression,filemode[:compression] library/tarfile,,:gz, @@ -239,9 +237,9 @@ library/urllib.request,,:password,"""joe:password@python.org""" library/urllib.parse,,:scheme, library/urllib.parse,,:scheme,URL:scheme://host/path library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 -library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" +library/venv,,:param,":param nodist: If true, setuptools and pip are not installed into the" library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" -library/venv,,:param,":param nopip: If True, pip is not installed into the created" +library/venv,,:param,":param nopip: If true, pip is not installed into the created" library/venv,,:param,:param context: The information for the virtual environment library/xmlrpc.client,,:nil,ex:nil library/xmlrpc.client,,:pass,http://user:pass@host:port/path @@ -333,6 +331,9 @@ library/xml.etree.elementtree,,:character,Commander Clement library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):" library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)" library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):" +library/xml.etree.elementtree,,:xi, +library/xml.etree.elementtree,,:include, +library/xml.etree.elementtree,,:include, Copyright (c) . library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" library/zipapp,,:fn,"pkg.mod:fn" library/zipapp,,:callable,"pkg.module:callable" @@ -352,4 +353,5 @@ whatsnew/changelog,,::,error::BytesWarning whatsnew/changelog,,::,default::BytesWarning whatsnew/changelog,,::,default::DeprecationWarning library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')" -library/importlib.metadata,,`,of directories ``path`` (defaults to sys.path). +library/importlib.metadata,,`,loading the metadata for packages for the indicated ``context``. +library/re,,`,"`" diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index fca44e9163cac7..21af621efce95b 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -4,7 +4,7 @@

{{ _('This Page') }}

  • {% trans %}Report a Bug{% endtrans %}
  • - {{ _('Show Source') }}
    • diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index d9364d6ced729f..987c63a7f3575c 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -40,7 +40,7 @@

    Download Python {{ release }} Documentation

    These archives contain all the content in the documentation.

    -

    HTML Help (.chm) files are made available in the "Windows" section +

    HTML Help (.chm) files are made available in the "Windows" section on the Python download page.

    diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 8d94137b01b519..3438b44377fcb9 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -5,3 +5,13 @@ {% trans %}CPython implementation detail:{% endtrans %} {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} + + +In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: + +{% trans %}in development{% endtrans %} +{% trans %}pre-release{% endtrans %} +{% trans %}stable{% endtrans %} +{% trans %}security-fixes{% endtrans %} +{% trans %}EOL{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 3666af92f0da47..f7bf6d8e491172 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,11 +2,8 @@

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    {% trans %}Docs by version{% endtrans %}

    diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index a765a5de8a2dfb..98ccf4224804b2 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -5,29 +5,21 @@
    {% trans %}This document is for an old version of Python that is no longer supported. You should upgrade, and read the {% endtrans %} - {% trans %} Python documentation for the last stable release {% endtrans %}. + {% trans %} Python documentation for the current stable release{% endtrans %}.
    {%- endif %} {% endblock %} {% block rootrellink %} {{ super() }} -
  • - {%- if switchers is defined %} - {{ language or 'en' }} - {{ release }} - {% trans %}Documentation {% endtrans %}{{ reldelim1 }} - {%- else %} +
  • {{ shorttitle }}{{ reldelim1 }} - {%- endif %}
    • {% endblock %} {% block extrahead %} {% if builder != "htmlhelp" %} - {% if switchers is defined and not embedded %} - {% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 7619ccbc1f36e9..0d780e3ba89643 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -114,8 +114,8 @@ accessible. "Directly accessible" here means that an unqualified reference to a name attempts to find the name in the namespace. Although scopes are determined statically, they are used dynamically. At any -time during execution, there are at least three nested scopes whose namespaces -are directly accessible: +time during execution, there are 3 or 4 nested scopes whose namespaces are +directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the @@ -143,10 +143,10 @@ language definition is evolving towards static name resolution, at "compile" time, so don't rely on dynamic name resolution! (In fact, local variables are already determined statically.) -A special quirk of Python is that -- if no :keyword:`global` statement is in -effect -- assignments to names always go into the innermost scope. Assignments -do not copy data --- they just bind names to objects. The same is true for -deletions: the statement ``del x`` removes the binding of ``x`` from the +A special quirk of Python is that -- if no :keyword:`global` or :keyword:`nonlocal` +statement is in effect -- assignments to names always go into the innermost scope. +Assignments do not copy data --- they just bind names to objects. The same is true +for deletions: the statement ``del x`` removes the binding of ``x`` from the namespace referenced by the local scope. In fact, all operations that introduce new names use the local scope: in particular, :keyword:`import` statements and function definitions bind the module or function name in the local scope. @@ -323,7 +323,7 @@ Instance Objects Now what can we do with instance objects? The only operations understood by instance objects are attribute references. There are two kinds of valid -attribute names, data attributes and methods. +attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, @@ -475,12 +475,20 @@ Random Remarks .. These should perhaps be placed more carefully... -Data attributes override method attributes with the same name; to avoid -accidental name conflicts, which may cause hard-to-find bugs in large programs, -it is wise to use some kind of convention that minimizes the chance of -conflicts. Possible conventions include capitalizing method names, prefixing -data attribute names with a small unique string (perhaps just an underscore), or -using verbs for methods and nouns for data attributes. +If the same attribute name occurs in both an instance and in a class, +then attribute lookup prioritizes the instance:: + + >>> class Warehouse: + purpose = 'storage' + region = 'west' + + >>> w1 = Warehouse() + >>> print(w1.purpose, w1.region) + storage west + >>> w2 = Warehouse() + >>> w2.region = 'east' + >>> print(w2.purpose, w2.region) + storage east Data attributes may be referenced by methods as well as by ordinary users ("clients") of an object. In other words, classes are not usable to implement @@ -841,7 +849,7 @@ defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: Generators ========== -:term:`Generator`\s are a simple and powerful tool for creating iterators. They +:term:`Generators ` are a simple and powerful tool for creating iterators. They are written like regular functions but use the :keyword:`yield` statement whenever they want to return data. Each time :func:`next` is called on it, the generator resumes where it left off (it remembers all the data values and which diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 79111f8518d9bb..284b68c0010aac 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -4,8 +4,8 @@ More Control Flow Tools *********************** -Besides the :keyword:`while` statement just introduced, Python knows the usual -control flow statements known from other languages, with some twists. +Besides the :keyword:`while` statement just introduced, Python uses the usual +flow control statements known from other languages, with some twists. .. _tut-if: @@ -66,20 +66,20 @@ they appear in the sequence. For example (no pun intended): window 6 defenestrate 12 -If you need to modify the sequence you are iterating over while inside the loop -(for example to duplicate selected items), it is recommended that you first -make a copy. Iterating over a sequence does not implicitly make a copy. The -slice notation makes this especially convenient:: +Code that modifies a collection while iterating over that same collection can +be tricky to get right. Instead, it is usually more straight-forward to loop +over a copy of the collection or to create a new collection:: - >>> for w in words[:]: # Loop over a slice copy of the entire list. - ... if len(w) > 6: - ... words.insert(0, w) - ... - >>> words - ['defenestrate', 'cat', 'window', 'defenestrate'] + # Strategy: Iterate over a copy + for user, status in users.copy().items(): + if status == 'inactive': + del users[user] -With ``for w in words:``, the example would attempt to create an infinite list, -inserting ``defenestrate`` over and over again. + # Strategy: Create a new collection + active_users = {} + for user, status in users.items(): + if status == 'active': + active_users[user] = status .. _tut-range: @@ -142,7 +142,7 @@ the list, thus saving space. We say such an object is :term:`iterable`, that is, suitable as a target for functions and constructs that expect something from which they can obtain successive items until the supply is exhausted. We have seen that -the :keyword:`for` statement is such a construct, while an example of function +the :keyword:`for` statement is such a construct, while an example of a function that takes an iterable is :func:`sum`:: >>> sum(range(4)) # 0 + 1 + 2 + 3 @@ -207,15 +207,15 @@ iteration of the loop:: ... if num % 2 == 0: ... print("Found an even number", num) ... continue - ... print("Found a number", num) + ... print("Found an odd number", num) Found an even number 2 - Found a number 3 + Found an odd number 3 Found an even number 4 - Found a number 5 + Found an odd number 5 Found an even number 6 - Found a number 7 + Found an odd number 7 Found an even number 8 - Found a number 9 + Found an odd number 9 .. _tut-pass: @@ -294,14 +294,14 @@ referenced. The actual parameters (arguments) to a function call are introduced in the local symbol table of the called function when it is called; thus, arguments are passed using *call by value* (where the *value* is always an object *reference*, -not the value of the object). [#]_ When a function calls another function, a new +not the value of the object). [#]_ When a function calls another function, +or calls itself recursively, a new local symbol table is created for that call. -A function definition introduces the function name in the current symbol table. -The value of the function name has a type that is recognized by the interpreter -as a user-defined function. This value can be assigned to another name which -can then also be used as a function. This serves as a general renaming -mechanism:: +A function definition associates the function name with the function object in +the current symbol table. The interpreter recognizes the object pointed to by +that name as a user-defined function. Other names can also point to that same +function object and can also be used to access the function:: >>> fib @@ -659,7 +659,7 @@ Finally, consider this function definition which has a potential collision betwe return 'name' in kwds There is no possible call that will make it return ``True`` as the keyword ``'name'`` -will always to bind to the first parameter. For example:: +will always bind to the first parameter. For example:: >>> foo(1, **{'name': 2}) Traceback (most recent call last): @@ -695,7 +695,7 @@ As guidance: * Use keyword-only when names have meaning and the function definition is more understandable by being explicit with names or you want to prevent users relying on the position of the argument being passed. -* For an API, use positional-only to prevent prevent breaking API changes +* For an API, use positional-only to prevent breaking API changes if the parameter's name is modified in the future. .. _tut-arbitraryargs: @@ -866,7 +866,7 @@ function. Parameter annotations are defined by a colon after the parameter name by an expression evaluating to the value of the annotation. Return annotations are defined by a literal ``->``, followed by an expression, between the parameter list and the colon denoting the end of the :keyword:`def` statement. The -following example has a positional argument, a keyword argument, and the return +following example has a required argument, an optional argument, and the return value annotated:: >>> def f(ham: str, eggs: str = 'eggs') -> str: @@ -920,7 +920,7 @@ extracted for you: bracketing constructs: ``a = f(1, 2) + g(3, 4)``. * Name your classes and functions consistently; the convention is to use - ``CamelCase`` for classes and ``lower_case_with_underscores`` for functions + ``UpperCamelCase`` for classes and ``lowercase_with_underscores`` for functions and methods. Always use ``self`` as the name for the first method argument (see :ref:`tut-firstclasses` for more on classes and methods). diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 01e437bb5da858..b0c2091a310808 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -78,7 +78,7 @@ objects: Return the number of times *x* appears in the list. -.. method:: list.sort(key=None, reverse=False) +.. method:: list.sort(*, key=None, reverse=False) :noindex: Sort the items of the list in place (the arguments can be used for sort @@ -125,6 +125,13 @@ only modify the list have no return value printed -- they return the default ``None``. [1]_ This is a design principle for all mutable data structures in Python. +Another thing you might notice is that not all data can be sorted or +compared. For instance, ``[None, 'hello', 10]`` doesn't sort because +integers can't be compared to strings and *None* can't be compared to +other types. Also, there are some types that don't have a defined +ordering relation. For example, ``3+4j < 5+7j`` isn't a valid +comparison. + .. _tut-lists-as-stacks: @@ -668,10 +675,11 @@ to a variable. For example, :: >>> non_null 'Trondheim' -Note that in Python, unlike C, assignment cannot occur inside expressions. C -programmers may grumble about this, but it avoids a common class of problems -encountered in C programs: typing ``=`` in an expression when ``==`` was -intended. +Note that in Python, unlike C, assignment inside expressions must be done +explicitly with the +:ref:`walrus operator ` ``:=``. +This avoids a common class of problems encountered in C programs: typing ``=`` +in an expression when ``==`` was intended. .. _tut-comparing: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 4e287bbd8d29ff..27c67c6e015af5 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -341,15 +341,46 @@ example:: File "", line 2, in KeyboardInterrupt -A *finally clause* is always executed before leaving the :keyword:`try` -statement, whether an exception has occurred or not. When an exception has -occurred in the :keyword:`!try` clause and has not been handled by an -:keyword:`except` clause (or it has occurred in an :keyword:`!except` or -:keyword:`!else` clause), it is re-raised after the :keyword:`finally` clause has -been executed. The :keyword:`!finally` clause is also executed "on the way out" -when any other clause of the :keyword:`!try` statement is left via a -:keyword:`break`, :keyword:`continue` or :keyword:`return` statement. A more -complicated example:: +If a :keyword:`finally` clause is present, the :keyword:`!finally` +clause will execute as the last task before the :keyword:`try` +statement completes. The :keyword:`!finally` clause runs whether or +not the :keyword:`!try` statement produces an exception. The following +points discuss more complex cases when an exception occurs: + +* If an exception occurs during execution of the :keyword:`!try` + clause, the exception may be handled by an :keyword:`except` + clause. If the exception is not handled by an :keyword:`!except` + clause, the exception is re-raised after the :keyword:`!finally` + clause has been executed. + +* An exception could occur during execution of an :keyword:`!except` + or :keyword:`!else` clause. Again, the exception is re-raised after + the :keyword:`!finally` clause has been executed. + +* If the :keyword:`!try` statement reaches a :keyword:`break`, + :keyword:`continue` or :keyword:`return` statement, the + :keyword:`!finally` clause will execute just prior to the + :keyword:`!break`, :keyword:`!continue` or :keyword:`!return` + statement's execution. + +* If a :keyword:`!finally` clause includes a :keyword:`!return` + statement, the returned value will be the one from the + :keyword:`!finally` clause's :keyword:`!return` statement, not the + value from the :keyword:`!try` clause's :keyword:`!return` + statement. + +For example:: + + >>> def bool_return(): + ... try: + ... return True + ... finally: + ... return False + ... + >>> bool_return() + False + +A more complicated example:: >>> def divide(x, y): ... try: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index fc2bd5578c4cf1..4e27cff83ce59f 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -172,7 +172,7 @@ Positional and keyword arguments can be arbitrarily combined:: If you have a really long format string that you don't want to split up, it would be nice if you could reference the variables to be formatted by name instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys :: +square brackets ``'[]'`` to access the keys. :: >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' @@ -257,10 +257,10 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The ``%`` operator can also be used for string formatting. It interprets the -left argument much like a :c:func:`sprintf`\ -style format string to be applied -to the right argument, and returns the string resulting from this formatting -operation. For example:: +The % operator (modulo) can also be used for string formatting. Given ``'string' +% values``, instances of ``%`` in ``string`` are replaced with zero or more +elements of ``values``. This operation is commonly known as string +interpolation. For example:: >>> import math >>> print('The value of pi is approximately %5.3f.' % math.pi) @@ -329,11 +329,16 @@ equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: If you're not using the :keyword:`with` keyword, then you should call ``f.close()`` to close the file and immediately free up any system -resources used by it. If you don't explicitly close a file, Python's -garbage collector will eventually destroy the object and close the -open file for you, but the file may stay open for a while. Another -risk is that different Python implementations will do this clean-up at -different times. +resources used by it. + +.. warning:: + Calling ``f.write()`` without using the :keyword:`!with` keyword or calling + ``f.close()`` **might** result in the arguments + of ``f.write()`` not being completely written to the disk, even if the + program exits successfully. + +.. + See also https://bugs.python.org/issue17852 After a file object is closed, either by a :keyword:`with` statement or by calling ``f.close()``, attempts to use the file object will @@ -358,8 +363,8 @@ To read a file's contents, call ``f.read(size)``, which reads some quantity of data and returns it as a string (in text mode) or bytes object (in binary mode). *size* is an optional numeric argument. When *size* is omitted or negative, the entire contents of the file will be read and returned; it's your problem if the -file is twice as large as your machine's memory. Otherwise, at most *size* bytes -are read and returned. +file is twice as large as your machine's memory. Otherwise, at most *size* +characters (in text mode) or *size* bytes (in binary mode) are read and returned. If the end of the file has been reached, ``f.read()`` will return an empty string (``''``). :: @@ -412,11 +417,11 @@ or a bytes object (in binary mode) -- before writing them:: represented as number of bytes from the beginning of the file when in binary mode and an opaque number when in text mode. -To change the file object's position, use ``f.seek(offset, from_what)``. The position is computed +To change the file object's position, use ``f.seek(offset, whence)``. The position is computed from adding *offset* to a reference point; the reference point is selected by -the *from_what* argument. A *from_what* value of 0 measures from the beginning +the *whence* argument. A *whence* value of 0 measures from the beginning of the file, 1 uses the current file position, and 2 uses the end of the file as -the reference point. *from_what* can be omitted and defaults to 0, using the +the reference point. *whence* can be omitted and defaults to 0, using the beginning of the file as the reference point. :: >>> f = open('workfile', 'rb+') diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index dddfd850cd29fb..501fb6a785624a 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -23,12 +23,10 @@ is an installation option, other places are possible; check with your local Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) -On Windows machines, the Python installation is usually placed in -:file:`C:\\Python38`, though you can change this when you're running the -installer. To add this directory to your path, you can type the following -command into :ref:`a command prompt window `:: - - set path=%path%;C:\python38 +On Windows machines where you have installed Python from the :ref:`Microsoft Store +`, the :file:`python3.8` command will be available. If you have +the :ref:`py.exe launcher ` installed, you can use the :file:`py` +command. See :ref:`setting-envvars` for other ways to launch Python. Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit @@ -36,13 +34,14 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support readline. Perhaps the -quickest check to see whether command line editing is supported is typing -:kbd:`Control-P` to the first Python prompt you get. If it beeps, you have command -line editing; see Appendix :ref:`tut-interacting` for an introduction to the -keys. If nothing appears to happen, or if ``^P`` is echoed, command line -editing isn't available; you'll only be able to use backspace to remove -characters from the current line. +substitution and code completion on systems that support the `GNU Readline +`_ library. +Perhaps the quickest check to see whether command line editing is supported is +typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you +have command line editing; see Appendix :ref:`tut-interacting` for an +introduction to the keys. If nothing appears to happen, or if ``^P`` is +echoed, command line editing isn't available; you'll only be able to use +backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard input connected to a tty device, it reads and executes commands interactively; diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index a4dbd6351b77d8..2a1666128a2015 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -394,7 +394,8 @@ indexed and sliced:: [9, 16, 25] All slice operations return a new list containing the requested elements. This -means that the following slice returns a new (shallow) copy of the list:: +means that the following slice returns a +:ref:`shallow copy ` of the list:: >>> squares[:] [1, 4, 9, 16, 25] diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 82261a6513489a..1b89056970852b 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -72,9 +72,22 @@ three`` at the command line:: >>> print(sys.argv) ['demo.py', 'one', 'two', 'three'] -The :mod:`getopt` module processes *sys.argv* using the conventions of the Unix -:func:`getopt` function. More powerful and flexible command line processing is -provided by the :mod:`argparse` module. +The :mod:`argparse` module provides a more sophisticated mechanism to process +command line arguments. The following script extracts one or more filenames +and an optional number of lines to be displayed:: + + import argparse + + parser = argparse.ArgumentParser(prog = 'top', + description = 'Show top lines from each file') + parser.add_argument('filenames', nargs='+') + parser.add_argument('-l', '--lines', type=int, default=10) + args = parser.parse_args() + print(args) + +When run at the command line with ``python top.py --lines=5 alpha.txt +beta.txt``, the script sets ``args.lines`` to ``5`` and ``args.filenames`` +to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index dc4136e42a887d..891dbdb97db1c7 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -50,6 +50,12 @@ This will create the ``tutorial-env`` directory if it doesn't exist, and also create directories inside it containing a copy of the Python interpreter, the standard library, and various supporting files. +A common directory location for a virtual environment is ``.venv``. +This name keeps the directory typically hidden in your shell and thus +out of the way while giving it a name that explains why the directory +exists. It also prevents clashing with ``.env`` environment variable +definition files that some tooling supports. + Once you've created a virtual environment, you may activate it. On Windows, run:: @@ -89,20 +95,9 @@ Managing Packages with pip You can install, upgrade, and remove packages using a program called :program:`pip`. By default ``pip`` will install packages from the Python Package Index, . You can browse the Python -Package Index by going to it in your web browser, or you can use ``pip``'s -limited search feature: - -.. code-block:: bash - - (tutorial-env) $ pip search astronomy - skyfield - Elegant astronomy for Python - gary - Galactic astronomy and gravitational dynamics. - novas - The United States Naval Observatory NOVAS astronomy library - astroobs - Provides astronomy ephemeris to plan telescope observations - PyAstronomy - A collection of astronomy related tools for Python. - ... +Package Index by going to it in your web browser. -``pip`` has a number of subcommands: "search", "install", "uninstall", +``pip`` has a number of subcommands: "install", "uninstall", "freeze", etc. (Consult the :ref:`installing-index` guide for complete documentation for ``pip``.) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 87af7e8be13376..08401d132009f7 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -70,6 +70,7 @@ source. :data:`sys.path` (allowing modules in that directory to be imported as top level modules). + .. audit-event:: cpython.run_command command cmdoption-c .. cmdoption:: -m @@ -100,11 +101,18 @@ source. first element will be set to ``"-m"``). As with the :option:`-c` option, the current directory will be added to the start of :data:`sys.path`. + :option:`-I` option can be used to run the script in isolated mode where + :data:`sys.path` contains neither the current directory nor the user's + site-packages directory. All :envvar:`PYTHON*` environment variables are + ignored, too. + Many standard library modules contain code that is invoked on their execution as a script. An example is the :mod:`timeit` module:: - python -mtimeit -s 'setup here' 'benchmarked code here' - python -mtimeit -h # for details + python -m timeit -s 'setup here' 'benchmarked code here' + python -m timeit -h # for details + + .. audit-event:: cpython.run_module module-name cmdoption-m .. seealso:: :func:`runpy.run_module` @@ -112,13 +120,13 @@ source. :pep:`338` -- Executing modules as scripts - .. versionchanged:: 3.1 Supply the package name to run a ``__main__`` submodule. .. versionchanged:: 3.4 namespace packages are also supported +.. _cmdarg-dash: .. describe:: - @@ -129,6 +137,9 @@ source. ``"-"`` and the current directory will be added to the start of :data:`sys.path`. + .. audit-event:: cpython.run_stdin "" "" + +.. _cmdarg-script: .. describe:: - - - - + + + + + - + - + -